From 8f972a8f727811aa14557ea59088d7b610477e46 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 19:45:26 +0200 Subject: [PATCH] [v10.2.x] InfluxDB: Fix table parsing with backend mode (#76990) InfluxDB: Fix table parsing with backend mode (#76899) * Add resultFormat to query model * don't add row name if the result format is table * No need special formatting since we use unified dataframes * betterer * specify visualization type in response * Unit tests * fix unit tests (cherry picked from commit 63abe25b744750197ddc870b1d89c4d881455bdc) Co-authored-by: ismail simsek --- .betterer.results | 7 +- pkg/tsdb/influxdb/influxql/response_parser.go | 34 ++++++-- .../influxdb/influxql/response_parser_test.go | 83 +++++++++++++++---- pkg/tsdb/influxdb/models/model_parser.go | 29 +++---- pkg/tsdb/influxdb/models/models.go | 29 +++---- .../plugins/datasource/influxdb/datasource.ts | 45 +--------- 6 files changed, 130 insertions(+), 97 deletions(-) diff --git a/.betterer.results b/.betterer.results index ebc179e8f69..82c54e4f042 100644 --- a/.betterer.results +++ b/.betterer.results @@ -6254,8 +6254,8 @@ exports[`better eslint`] = { [0, 0, 0, "Styles should be written using objects.", "0"] ], "public/app/plugins/datasource/influxdb/datasource.ts:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"], + [0, 0, 0, "Do not use any type assertions.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "1"], [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"], [0, 0, 0, "Unexpected any. Specify a different type.", "4"], @@ -6271,8 +6271,7 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "14"], [0, 0, 0, "Unexpected any. Specify a different type.", "15"], [0, 0, 0, "Unexpected any. Specify a different type.", "16"], - [0, 0, 0, "Unexpected any. Specify a different type.", "17"], - [0, 0, 0, "Do not use any type assertions.", "18"] + [0, 0, 0, "Do not use any type assertions.", "17"] ], "public/app/plugins/datasource/influxdb/influx_query_model.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/pkg/tsdb/influxdb/influxql/response_parser.go b/pkg/tsdb/influxdb/influxql/response_parser.go index 717774508db..07471d11bb0 100644 --- a/pkg/tsdb/influxdb/influxql/response_parser.go +++ b/pkg/tsdb/influxdb/influxql/response_parser.go @@ -19,6 +19,12 @@ var ( legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`) ) +const ( + graphVisType data.VisType = "graph" + tableVisType data.VisType = "table" + logsVisType data.VisType = "logs" +) + func ResponseParse(buf io.ReadCloser, statusCode int, query *models.Query) *backend.DataResponse { return parse(buf, statusCode, query) } @@ -155,7 +161,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo name := string(formatFrameName(row, column, query, frameName[:])) valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: name}) - return newDataFrame(name, query.RawQuery, timeField, valueField) + return newDataFrame(name, query.RawQuery, timeField, valueField, getVisType(query.ResultFormat)) } func newFrameWithoutTimeField(row models.Row, retentionPolicyQuery bool, tagValuesQuery bool) *data.Frame { @@ -198,10 +204,11 @@ func newFrameWithoutTimeField(row models.Row, retentionPolicyQuery bool, tagValu return data.NewFrame(row.Name, field) } -func newDataFrame(name string, queryString string, timeField *data.Field, valueField *data.Field) *data.Frame { +func newDataFrame(name string, queryString string, timeField *data.Field, valueField *data.Field, visType data.VisType) *data.Frame { frame := data.NewFrame(name, timeField, valueField) frame.Meta = &data.FrameMeta{ - ExecutedQueryString: queryString, + ExecutedQueryString: queryString, + PreferredVisualization: visType, } return frame @@ -209,7 +216,7 @@ func newDataFrame(name string, queryString string, timeField *data.Field, valueF func formatFrameName(row models.Row, column string, query models.Query, frameName []byte) []byte { if query.Alias == "" { - return buildFrameNameFromQuery(row, column, frameName) + return buildFrameNameFromQuery(row, column, frameName, query.ResultFormat) } nameSegment := strings.Split(row.Name, ".") @@ -247,9 +254,11 @@ func formatFrameName(row models.Row, column string, query models.Query, frameNam return result } -func buildFrameNameFromQuery(row models.Row, column string, frameName []byte) []byte { - frameName = append(frameName, row.Name...) - frameName = append(frameName, '.') +func buildFrameNameFromQuery(row models.Row, column string, frameName []byte, resultFormat string) []byte { + if resultFormat != "table" { + frameName = append(frameName, row.Name...) + frameName = append(frameName, '.') + } frameName = append(frameName, column...) if len(row.Tags) > 0 { @@ -323,6 +332,17 @@ func parseNumber(value any) *float64 { return &fvalue } +func getVisType(resFormat string) data.VisType { + switch resFormat { + case "table": + return tableVisType + case "logs": + return logsVisType + default: + return graphVisType + } +} + func isTagValuesQuery(query models.Query) bool { return strings.Contains(strings.ToLower(query.RawQuery), strings.ToLower("SHOW TAG VALUES")) } diff --git a/pkg/tsdb/influxdb/influxql/response_parser_test.go b/pkg/tsdb/influxdb/influxql/response_parser_test.go index a47b9fb36ea..cf2da76ed36 100644 --- a/pkg/tsdb/influxdb/influxql/response_parser_test.go +++ b/pkg/tsdb/influxdb/influxql/response_parser_test.go @@ -28,6 +28,10 @@ func generateQuery(query models.Query) *models.Query { if query.RawQuery == "" { query.RawQuery = "Test raw query" } + + if query.ResultFormat == "" { + query.ResultFormat = "time_series" + } return &query } @@ -82,7 +86,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), floatField, ) - floatFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + floatFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} string_test := "/usr/path" stringField := data.NewField("Value", labels, []*string{ @@ -98,7 +102,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), stringField, ) - stringFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + stringFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} bool_true := true bool_false := false @@ -115,7 +119,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), boolField, ) - boolFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + boolFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(prepare(response), 200, generateQuery(query)) @@ -265,7 +269,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(prepare(response), 200, generateQuery(query)) @@ -310,7 +314,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(prepare(response), 200, generateQuery(query)) @@ -359,7 +363,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(prepare(response), 200, generateQuery(query)) t.Run("should parse aliases", func(t *testing.T) { if diff := cmp.Diff(testFrame, result.Frames[0], data.FrameTestCompareOptions()...); diff != "" { @@ -586,7 +590,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(prepare(response), 200, generateQuery(query)) require.EqualError(t, result.Error, "query-timeout limit exceeded") @@ -707,12 +711,14 @@ func TestResponseParser_Parse_RetentionPolicy(t *testing.T) { func TestResponseParser_Parse(t *testing.T) { tests := []struct { - name string - input string - f func(t *testing.T, got backend.DataResponse) + name string + resFormat string + input string + f func(t *testing.T, got backend.DataResponse) }{ { - name: "Influxdb response parser with valid value when null values returned", + name: "Influxdb response parser with valid value when null values returned", + resFormat: "time_series", input: `{ "results": [ { "series": [ { "name": "cpu", "columns": ["time","mean"], @@ -734,12 +740,13 @@ func TestResponseParser_Parse(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} assert.Equal(t, testFrame, got.Frames[0]) }, }, { - name: "Influxdb response parser with valid value when all values are null", + name: "Influxdb response parser with valid value when all values are null", + resFormat: "time_series", input: `{ "results": [ { "series": [ { "name": "cpu", "columns": ["time","mean"], @@ -761,14 +768,60 @@ func TestResponseParser_Parse(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} assert.Equal(t, testFrame, got.Frames[0]) }, }, + { + name: "Influxdb response parser with table result", + resFormat: "table", + input: `{ + "results": [ + { + "statement_id": 0, + "series": [ + { + "name": "Annotation", + "columns": [ + "time", + "domain", + "type", + "ASD", + "details" + ], + "values": [ + [ + 1697789142916, + "AASD157", + "fghg", + null, + "Something happened AtTime=2023-10-20T08:05:42.902036" + ], + [ + 1697789142918, + "HUY23", + "val23", + null, + "Something else happened AtTime=2023-10-20T08:05:42.902036" + ] + ] + } + ] + } + ] + }`, + f: func(t *testing.T, got backend.DataResponse) { + assert.Equal(t, "domain", got.Frames[0].Name) + assert.Equal(t, "domain", got.Frames[0].Fields[1].Config.DisplayNameFromDS) + assert.Equal(t, "ASD", got.Frames[2].Name) + assert.Equal(t, "ASD", got.Frames[2].Fields[1].Config.DisplayNameFromDS) + assert.Equal(t, tableVisType, got.Frames[0].Meta.PreferredVisualization) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := ResponseParse(prepare(tt.input), 200, generateQuery(models.Query{})) + got := ResponseParse(prepare(tt.input), 200, generateQuery(models.Query{ResultFormat: tt.resFormat})) require.NotNil(t, got) if tt.f != nil { tt.f(t, *got) diff --git a/pkg/tsdb/influxdb/models/model_parser.go b/pkg/tsdb/influxdb/models/model_parser.go index 4396c504a3b..06178f9f85b 100644 --- a/pkg/tsdb/influxdb/models/model_parser.go +++ b/pkg/tsdb/influxdb/models/model_parser.go @@ -26,8 +26,8 @@ func QueryParse(query backend.DataQuery) (*Query, error) { limit := model.Get("limit").MustString("") slimit := model.Get("slimit").MustString("") orderByTime := model.Get("orderByTime").MustString("") - measurement := model.Get("measurement").MustString("") + resultFormat := model.Get("resultFormat").MustString("") tags, err := parseTags(model) if err != nil { @@ -54,19 +54,20 @@ func QueryParse(query backend.DataQuery) (*Query, error) { } return &Query{ - Measurement: measurement, - Policy: policy, - GroupBy: groupBys, - Tags: tags, - Selects: selects, - RawQuery: rawQuery, - Interval: interval, - Alias: alias, - UseRawQuery: useRawQuery, - Tz: tz, - Limit: limit, - Slimit: slimit, - OrderByTime: orderByTime, + Measurement: measurement, + Policy: policy, + GroupBy: groupBys, + Tags: tags, + Selects: selects, + RawQuery: rawQuery, + Interval: interval, + Alias: alias, + UseRawQuery: useRawQuery, + Tz: tz, + Limit: limit, + Slimit: slimit, + OrderByTime: orderByTime, + ResultFormat: resultFormat, }, nil } diff --git a/pkg/tsdb/influxdb/models/models.go b/pkg/tsdb/influxdb/models/models.go index a4c67645995..020b90742bd 100644 --- a/pkg/tsdb/influxdb/models/models.go +++ b/pkg/tsdb/influxdb/models/models.go @@ -3,20 +3,21 @@ package models import "time" type Query struct { - Measurement string - Policy string - Tags []*Tag - GroupBy []*QueryPart - Selects []*Select - RawQuery string - UseRawQuery bool - Alias string - Interval time.Duration - Tz string - Limit string - Slimit string - OrderByTime string - RefID string + Measurement string + Policy string + Tags []*Tag + GroupBy []*QueryPart + Selects []*Select + RawQuery string + UseRawQuery bool + Alias string + Interval time.Duration + Tz string + Limit string + Slimit string + OrderByTime string + RefID string + ResultFormat string } type Tag struct { diff --git a/public/app/plugins/datasource/influxdb/datasource.ts b/public/app/plugins/datasource/influxdb/datasource.ts index ba6a4f2a4d7..246ecdc9ec7 100644 --- a/public/app/plugins/datasource/influxdb/datasource.ts +++ b/public/app/plugins/datasource/influxdb/datasource.ts @@ -1,4 +1,4 @@ -import { cloneDeep, extend, groupBy, has, isString, map as _map, omit, pick, reduce } from 'lodash'; +import { cloneDeep, extend, has, isString, map as _map, omit, pick, reduce } from 'lodash'; import { lastValueFrom, merge, Observable, of, throwError } from 'rxjs'; import { catchError, map } from 'rxjs/operators'; @@ -138,49 +138,8 @@ export default class InfluxDatasource extends DataSourceWithBackend { - if (res.error) { - throw { - message: 'InfluxDB Error: ' + res.error.message, - res, - }; - } - - const groupedFrames = groupBy(res.data, (x) => x.refId); - if (Object.keys(groupedFrames).length === 0) { - return { data: [] }; - } - - const seriesList: any[] = []; - filteredRequest.targets.forEach((target) => { - const filteredFrames = groupedFrames[target.refId] ?? []; - switch (target.resultFormat) { - case 'logs': - case 'table': - seriesList.push( - this.responseParser.getTable(filteredFrames, target, { - preferredVisualisationType: target.resultFormat, - }) - ); - break; - default: { - for (let i = 0; i < filteredFrames.length; i++) { - seriesList.push(filteredFrames[i]); - } - break; - } - } - }); - - return { data: seriesList }; - }) - ); + return super.query(filteredRequest); } // Fallback to classic query support