From 808cf75ff87fef1612d6c104dac9fb8250e1ebff Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 6 Jun 2024 19:14:53 +0200 Subject: [PATCH] InfluxDB: Use json-iterator package for json operations (#88562) * return error early * enable gzip between grafana and influxdb * use json-iterator package for json operations * revert gzip changes * update test * go mod tidy go work sync --- go.sum | 7 ++++ go.work.sum | 6 +-- .../influxql/buffered/response_parser.go | 40 ++++++++++++------- .../influxql/buffered/response_parser_test.go | 5 ++- pkg/tsdb/influxdb/influxql/util/util.go | 8 +--- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/go.sum b/go.sum index 91eddaa3dd1..138b3a68ae7 100644 --- a/go.sum +++ b/go.sum @@ -3474,6 +3474,7 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -3552,6 +3553,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -3653,6 +3655,7 @@ golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3871,6 +3874,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -3897,6 +3901,7 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -3919,6 +3924,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -4028,6 +4034,7 @@ golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go.work.sum b/go.work.sum index 1ed8e444a88..53a096993b2 100644 --- a/go.work.sum +++ b/go.work.sum @@ -975,20 +975,16 @@ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/image v0.0.0-20220302094943-723b81ca9867 h1:TcHcE0vrmgzNH1v3ppjcMGbhG5+9fMuvOmUYwNEF4q4= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808 h1:+Kc94D8UVEVxJnLXp/+FMfqQARZtWHfVrcRtcG8aT3g= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= diff --git a/pkg/tsdb/influxdb/influxql/buffered/response_parser.go b/pkg/tsdb/influxdb/influxql/buffered/response_parser.go index f0ee1d167fe..fbd93486511 100644 --- a/pkg/tsdb/influxdb/influxql/buffered/response_parser.go +++ b/pkg/tsdb/influxdb/influxql/buffered/response_parser.go @@ -1,7 +1,6 @@ package buffered import ( - "encoding/json" "fmt" "io" "strings" @@ -9,6 +8,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + jsoniter "github.com/json-iterator/go" "github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/util" "github.com/grafana/grafana/pkg/tsdb/influxdb/models" @@ -46,8 +46,9 @@ func parse(buf io.Reader, statusCode int, query *models.Query) *backend.DataResp func parseJSON(buf io.Reader) (models.Response, error) { var response models.Response + json := jsoniter.ConfigCompatibleWithStandardLibrary + dec := json.NewDecoder(buf) - dec.UseNumber() err := dec.Decode(&response) @@ -150,6 +151,12 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn case "json.Number": value := util.ParseNumber(valuePair[colIdx]) floatArray = append(floatArray, value) + case "float64": + if value, ok := valuePair[colIdx].(float64); ok { + floatArray = append(floatArray, &value) + } else { + floatArray = append(floatArray, nil) + } case "bool": value, ok := valuePair[colIdx].(bool) if ok { @@ -192,6 +199,8 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn valueField = data.NewField(row.Columns[colIdx], labels, stringArray) case "json.Number": valueField = data.NewField(row.Columns[colIdx], labels, floatArray) + case "float64": + valueField = data.NewField(row.Columns[colIdx], labels, floatArray) case "bool": valueField = data.NewField(row.Columns[colIdx], labels, boolArray) case "null": @@ -237,12 +246,6 @@ func transformRowsForTimeSeries(rows []models.Row, query models.Query) data.Fram if !hasTimeCol { newFrame := newFrameWithoutTimeField(row, query) - if len(frames) == 0 { - newFrame.Meta = &data.FrameMeta{ - ExecutedQueryString: query.RawQuery, - PreferredVisualization: util.GetVisType(query.ResultFormat), - } - } frames = append(frames, newFrame) } else { for colIndex, column := range row.Columns { @@ -250,17 +253,18 @@ func transformRowsForTimeSeries(rows []models.Row, query models.Query) data.Fram continue } newFrame := newFrameWithTimeField(row, column, colIndex, query, frameName) - if len(frames) == 0 { - newFrame.Meta = &data.FrameMeta{ - ExecutedQueryString: query.RawQuery, - PreferredVisualization: util.GetVisType(query.ResultFormat), - } - } frames = append(frames, newFrame) } } } + if len(frames) > 0 { + frames[0].Meta = &data.FrameMeta{ + ExecutedQueryString: query.RawQuery, + PreferredVisualization: util.GetVisType(query.ResultFormat), + } + } + return frames } @@ -290,6 +294,12 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo case "json.Number": value := util.ParseNumber(valuePair[colIndex]) floatArray = append(floatArray, value) + case "float64": + if value, ok := valuePair[colIndex].(float64); ok { + floatArray = append(floatArray, &value) + } else { + floatArray = append(floatArray, nil) + } case "bool": value, ok := valuePair[colIndex].(bool) if ok { @@ -311,6 +321,8 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo valueField = data.NewField("Value", row.Tags, stringArray) case "json.Number": valueField = data.NewField("Value", row.Tags, floatArray) + case "float64": + valueField = data.NewField("Value", row.Tags, floatArray) case "bool": valueField = data.NewField("Value", row.Tags, boolArray) case "null": diff --git a/pkg/tsdb/influxdb/influxql/buffered/response_parser_test.go b/pkg/tsdb/influxdb/influxql/buffered/response_parser_test.go index 890d25a4125..3dab1ff1603 100644 --- a/pkg/tsdb/influxdb/influxql/buffered/response_parser_test.go +++ b/pkg/tsdb/influxdb/influxql/buffered/response_parser_test.go @@ -380,10 +380,11 @@ func TestInfluxdbResponseParser(t *testing.T) { } }) - t.Run("Influxdb response parser parseTimestamp valid JSON.number", func(t *testing.T) { + t.Run("Influxdb response parser parseTimestamp valid number", func(t *testing.T) { // currently we use milliseconds-precision with influxdb, so the test works with that. // if we change this to for example nanoseconds-precision, the tests will have to change. - timestamp, err := util.ParseTimestamp(json.Number("1609556645000")) + ts := float64(1609556645000) + timestamp, err := util.ParseTimestamp(ts) require.NoError(t, err) require.Equal(t, timestamp.Format(time.RFC3339), "2021-01-02T03:04:05Z") }) diff --git a/pkg/tsdb/influxdb/influxql/util/util.go b/pkg/tsdb/influxdb/influxql/util/util.go index 33ddfdff845..06ed3cb67a4 100644 --- a/pkg/tsdb/influxdb/influxql/util/util.go +++ b/pkg/tsdb/influxdb/influxql/util/util.go @@ -98,18 +98,14 @@ func BuildFrameNameFromQuery(rowName, column string, tags map[string]string, fra } func ParseTimestamp(value any) (time.Time, error) { - timestampNumber, ok := value.(json.Number) + timestampNumber, ok := value.(float64) if !ok { return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value) } - timestampInMilliseconds, err := timestampNumber.Int64() - if err != nil { - return time.Time{}, err - } // currently in the code the influxdb-timestamps are requested with // milliseconds-precision, meaning these values are milliseconds - t := time.UnixMilli(timestampInMilliseconds).UTC() + t := time.UnixMilli(int64(timestampNumber)).UTC() return t, nil }