Prometheus: Improve response parser performance (#101517)
* introduce tests
* improve allocation by %61
* ~%27 improvement
goos: darwin
goarch: arm64
pkg: github.com/grafana/grafana/pkg/promlib/converter
cpu: Apple M1 Pro
│ pmem.0.txt │ pmem.2.txt │
│ sec/op │ sec/op vs base │
ReadPrometheusStyleResult_FromFile/prom-query-range.json-10 117.72µ ± 6% 82.90µ ± 4% -29.57% (p=0.002 n=6)
│ pmem.0.txt │ pmem.2.txt │
│ B/op │ B/op vs base │
ReadPrometheusStyleResult_FromFile/prom-query-range.json-10 104.34Ki ± 0% 76.09Ki ± 0% -27.08% (p=0.002 n=6)
│ pmem.0.txt │ pmem.2.txt │
│ allocs/op │ allocs/op vs base │
ReadPrometheusStyleResult_FromFile/prom-query-range.json-10 2463.0 ± 0% 899.0 ± 0% -63.50% (p=0.002 n=6)
* add more tests
* remove comment lines
* read string as slice to prevent type conversion
* golang lint
* revert appendValueFromString improvement
* fix merging issues
This commit is contained in:
ismail simsek authored and 
GitHub committed
1 parent
df0df68f30
commit
f64be062c4
7 files changed
+99718
-21
No files matched your search
@@ -597,17 +597,17 @@ func readScalar(iter *sdkjsoniter.Iterator, dataPlane bool) backend.DataResponse
|
||||
|
||||
func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt Options) backend.DataResponse {
|
||||
rsp := backend.DataResponse{}
|
||||
size := 0
|
||||
|
||||
for more, err := iter.ReadArray(); more; more, err = iter.ReadArray() {
|
||||
if err != nil {
|
||||
return rspErr(err)
|
||||
}
|
||||
timeField := data.NewFieldFromFieldType(data.FieldTypeTime, 0)
|
||||
timeField.Name = data.TimeSeriesTimeFieldName
|
||||
valueField := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
|
||||
valueField.Name = data.TimeSeriesValueFieldName
|
||||
valueField.Labels = data.Labels{}
|
||||
|
||||
// First read all values to temporary storage
|
||||
tempTimes := make([]time.Time, 0, size)
|
||||
tempValues := make([]float64, 0, size)
|
||||
var labels data.Labels
|
||||
var histogram *histogramInfo
|
||||
|
||||
for l1Field, err := iter.ReadObject(); l1Field != ""; l1Field, err = iter.ReadObject() {
|
||||
@@ -616,7 +616,7 @@ func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt
|
||||
}
|
||||
switch l1Field {
|
||||
case "metric":
|
||||
if err = iter.ReadVal(&valueField.Labels); err != nil {
|
||||
if err = iter.ReadVal(&labels); err != nil {
|
||||
return rspErr(err)
|
||||
}
|
||||
|
||||
@@ -625,10 +625,9 @@ func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt
|
||||
if err != nil {
|
||||
return rspErr(err)
|
||||
}
|
||||
timeField.Append(t)
|
||||
valueField.Append(v)
|
||||
tempTimes = append(tempTimes, t)
|
||||
tempValues = append(tempValues, v)
|
||||
|
||||
// nolint:goconst
|
||||
case "values":
|
||||
for more, err := iter.ReadArray(); more; more, err = iter.ReadArray() {
|
||||
if err != nil {
|
||||
@@ -638,8 +637,8 @@ func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt
|
||||
if err != nil {
|
||||
return rspErr(err)
|
||||
}
|
||||
timeField.Append(t)
|
||||
valueField.Append(v)
|
||||
tempTimes = append(tempTimes, t)
|
||||
tempValues = append(tempValues, v)
|
||||
}
|
||||
|
||||
case "histogram":
|
||||
@@ -673,21 +672,18 @@ func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt
|
||||
}
|
||||
|
||||
if histogram != nil {
|
||||
histogram.yMin.Labels = valueField.Labels
|
||||
histogram.yMax.Labels = valueField.Labels
|
||||
histogram.count.Labels = valueField.Labels
|
||||
histogram.yLayout.Labels = valueField.Labels
|
||||
histogram.time.Labels = valueField.Labels
|
||||
frame := data.NewFrame(valueField.Name, histogram.time, histogram.yMin, histogram.yMax, histogram.count, histogram.yLayout)
|
||||
histogram.yMin.Labels = labels
|
||||
histogram.yMax.Labels = labels
|
||||
histogram.count.Labels = labels
|
||||
histogram.yLayout.Labels = labels
|
||||
histogram.time.Labels = labels
|
||||
frame := data.NewFrame("", histogram.time, histogram.yMin, histogram.yMax, histogram.count, histogram.yLayout)
|
||||
frame.Meta = &data.FrameMeta{
|
||||
Type: "heatmap-cells",
|
||||
}
|
||||
if frame.Name == data.TimeSeriesValueFieldName {
|
||||
frame.Name = "" // only set the name if useful
|
||||
}
|
||||
rsp.Frames = append(rsp.Frames, frame)
|
||||
} else {
|
||||
frame := data.NewFrame("", timeField, valueField)
|
||||
frame := data.NewFrame("", data.NewField(data.TimeSeriesTimeFieldName, nil, tempTimes), data.NewField(data.TimeSeriesValueFieldName, labels, tempValues))
|
||||
frame.Meta = &data.FrameMeta{
|
||||
Type: data.FrameTypeTimeSeriesMulti,
|
||||
Custom: resultTypeToCustomMeta(resultType),
|
||||
@@ -699,6 +695,7 @@ func readMatrixOrVectorMulti(iter *sdkjsoniter.Iterator, resultType string, opt
|
||||
frame.Meta.TypeVersion = data.FrameTypeVersion{0, 1}
|
||||
}
|
||||
rsp.Frames = append(rsp.Frames, frame)
|
||||
size = len(tempTimes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package converter
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
// readTestData reads a JSON file from testdata directory
|
||||
func readTestData(t *testing.B, filename string) []byte {
|
||||
// Can ignore gosec G304 here, because this is a constant defined below benchmark test
|
||||
// nolint:gosec
|
||||
data, err := os.ReadFile("testdata/" + filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// BenchmarkReadPrometheusStyleResult_FromFile benchmarks processing different test files
|
||||
// go test -benchmem -run=^$ -bench=BenchmarkReadPrometheusStyleResult_FromFile$ github.com/grafana/grafana/pkg/promlib/converter/ -memprofile pmem.out -count 6 | tee pmem.0.txt
|
||||
func BenchmarkReadPrometheusStyleResult_FromFile(b *testing.B) {
|
||||
testFiles := []string{
|
||||
"prom-query-range.json",
|
||||
"prom-query-range-big.json",
|
||||
"prom-matrix-histogram-partitioned.json",
|
||||
}
|
||||
|
||||
opt := Options{Dataplane: true}
|
||||
|
||||
for _, tf := range testFiles {
|
||||
testData := readTestData(b, tf)
|
||||
iter := jsoniter.ParseBytes(jsoniter.ConfigDefault, testData)
|
||||
|
||||
b.Run(tf, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = ReadPrometheusStyleResult(iter, opt)
|
||||
iter.ResetBytes(testData)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,8 @@ var files = []string{
|
||||
"prom-exemplars-a",
|
||||
"prom-exemplars-b",
|
||||
"prom-exemplars-diff-labels",
|
||||
"prom-query-range",
|
||||
"prom-query-range-big",
|
||||
"loki-streams-a",
|
||||
"loki-streams-b",
|
||||
"loki-streams-c",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+62468
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"process_cpu_seconds_total","instance":"prometheus-high-card:9111","job":"prometheus-high-card"},"values":[[1740338424.615,"5608.51"],[1740338438.615,"5609.06"],[1740338452.615,"5609.61"],[1740338466.615,"5610.64"],[1740338480.615,"5611.15"],[1740338494.615,"5612.23"],[1740338508.615,"5612.69"],[1740338522.615,"5613.19"],[1740338536.615,"5614.12"],[1740338550.615,"5614.61"],[1740338564.615,"5615.46"],[1740338578.615,"5615.88"],[1740338592.615,"5616.42"],[1740338606.615,"5617.65"],[1740338620.615,"5618.22"],[1740338634.615,"5619.52"],[1740338648.615,"5620.12"],[1740338662.615,"5620.72"],[1740338676.615,"5621.88"],[1740338690.615,"5622.45"],[1740338704.615,"5623.59"],[1740338718.615,"5624.19"],[1740338732.615,"5624.69"],[1740338746.615,"5625.92"],[1740338760.615,"5626.58"],[1740338774.615,"5627.84"],[1740338788.615,"5628.49"],[1740338802.615,"5629.1"],[1740338816.615,"5630.28"],[1740338830.615,"5630.86"],[1740338844.615,"5632.05"],[1740338858.615,"5632.65"],[1740338872.615,"5633.3"],[1740338886.615,"5634.62"],[1740338900.615,"5635.25"],[1740338914.615,"5636.5"],[1740338928.615,"5637.12"],[1740338942.615,"5637.71"],[1740338956.615,"5638.96"],[1740338970.615,"5639.55"],[1740338984.615,"5640.61"],[1740338998.615,"5641.15"],[1740339012.615,"5641.72"],[1740339026.615,"5642.9"],[1740339040.615,"5643.51"],[1740339054.615,"5644.79"],[1740339068.615,"5645.4"],[1740339082.615,"5646.03"],[1740339096.615,"5647.3"],[1740339110.615,"5647.88"],[1740339124.615,"5649.06"],[1740339138.615,"5649.63"],[1740339152.615,"5650.3"],[1740339166.615,"5651.56"],[1740339180.615,"5652.15"],[1740339194.615,"5653.37"],[1740339208.615,"5653.98"],[1740339222.615,"5654.59"],[1740339236.615,"5656.03"],[1740339250.615,"5656.65"],[1740339264.615,"5657.97"],[1740339278.615,"5658.55"],[1740339292.615,"5658.97"],[1740339306.615,"5659.86"],[1740339320.615,"5660.42"],[1740339334.615,"5661.64"],[1740339348.615,"5662.27"],[1740339362.615,"5662.9"],[1740339376.615,"5664.09"],[1740339390.615,"5664.64"],[1740339404.615,"5665.92"],[1740339418.615,"5666.53"],[1740339432.615,"5667.11"],[1740339446.615,"5668.34"],[1740339460.615,"5668.91"],[1740339474.615,"5670.06"],[1740339488.615,"5670.6"],[1740339502.615,"5671.19"],[1740339516.615,"5672.44"],[1740339530.615,"5672.99"],[1740339544.615,"5674.21"],[1740339558.615,"5674.81"],[1740339572.615,"5675.41"],[1740339586.615,"5676.71"],[1740339600.615,"5677.36"],[1740339614.615,"5678.54"],[1740339628.615,"5679.09"],[1740339642.615,"5679.68"],[1740339656.615,"5680.77"],[1740339670.615,"5681.24"],[1740339684.615,"5682.29"],[1740339698.615,"5682.85"],[1740339712.615,"5683.35"],[1740339726.615,"5684.36"],[1740339740.615,"5684.85"],[1740339754.615,"5685.86"],[1740339768.615,"5686.35"],[1740339782.615,"5686.88"],[1740339796.615,"5688"],[1740339810.615,"5688.57"],[1740339824.615,"5689.7"],[1740339838.615,"5690.23"],[1740339852.615,"5690.75"],[1740339866.615,"5691.74"],[1740339880.615,"5692.18"],[1740339894.615,"5693.01"],[1740339908.615,"5693.63"],[1740339922.615,"5694.23"],[1740339936.615,"5695.49"],[1740339950.615,"5696.06"],[1740339964.615,"5697.17"],[1740339978.615,"5697.75"],[1740339992.615,"5698.29"],[1740340006.615,"5699.32"],[1740340020.615,"5699.85"],[1740340034.615,"5701.04"],[1740340048.615,"5701.62"],[1740340062.615,"5702.23"],[1740340076.615,"5703.38"],[1740340090.615,"5703.92"],[1740340104.615,"5705.01"],[1740340118.615,"5705.55"],[1740340132.615,"5706.04"],[1740340146.615,"5707.11"],[1740340160.615,"5707.69"],[1740340174.615,"5708.9"],[1740340188.615,"5709.52"],[1740340202.615,"5710.12"],[1740340216.615,"5711.19"],[1740340230.615,"5711.69"],[1740340244.615,"5712.68"],[1740340258.615,"5713.25"],[1740340272.615,"5713.92"],[1740340286.615,"5715.19"],[1740340300.615,"5715.76"],[1740340314.615,"5716.89"],[1740340328.615,"5717.49"],[1740340342.615,"5718.05"],[1740340356.615,"5719.15"],[1740340370.615,"5719.74"],[1740340384.615,"5720.72"],[1740340398.615,"5721.4"],[1740340412.615,"5722.09"],[1740340426.615,"5723.42"],[1740340440.615,"5724.1"],[1740340454.615,"5725.35"],[1740340468.615,"5725.96"],[1740340482.615,"5726.62"],[1740340496.615,"5727.84"],[1740340510.615,"5728.34"],[1740340524.615,"5729.47"],[1740340538.615,"5730.12"],[1740340552.615,"5730.72"],[1740340566.615,"5731.98"],[1740340580.615,"5732.54"],[1740340594.615,"5733.57"],[1740340608.615,"5734.08"],[1740340622.615,"5734.61"],[1740340636.615,"5735.64"],[1740340650.615,"5736.14"],[1740340664.615,"5737.25"],[1740340678.615,"5737.83"],[1740340692.615,"5738.41"],[1740340706.615,"5739.66"],[1740340720.615,"5740.27"],[1740340734.615,"5741.43"],[1740340748.615,"5742.02"],[1740340762.615,"5742.57"],[1740340776.615,"5743.72"],[1740340790.615,"5744.28"],[1740340804.615,"5745.37"],[1740340818.615,"5745.93"],[1740340832.615,"5746.5"],[1740340846.615,"5747.63"],[1740340860.615,"5748.16"],[1740340874.615,"5749.31"],[1740340888.615,"5749.88"],[1740340902.615,"5750.46"],[1740340916.615,"5751Line truncated
|
||||
Reference in new issue
Block a user