From 3bd7217aa2149348deee1a83c548526760c2c90b Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Tue, 7 Mar 2023 13:56:34 +0100 Subject: [PATCH] Elasticsearch: Remove _source field when processing raw data on backend (#64119) * Elasticsearch: Remove _source field when processing raw data on backend * Update snapshot test --- pkg/tsdb/elasticsearch/response_parser.go | 1 - .../response_parser_frontend_test.go | 47 ------ .../elasticsearch/response_parser_test.go | 58 ++++++- .../testdata_response/raw_data.a.golden.jsonc | 155 ++---------------- 4 files changed, 64 insertions(+), 197 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index ef44bd84ca3..a1952361c6a 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -154,7 +154,6 @@ func processRawDataResponse(res *es.SearchResponse, target *Query, configuredFie "_index": hit["_index"], "sort": hit["sort"], "highlight": hit["highlight"], - "_source": flattened, } for k, v := range flattened { diff --git a/pkg/tsdb/elasticsearch/response_parser_frontend_test.go b/pkg/tsdb/elasticsearch/response_parser_frontend_test.go index 285c8755898..9b831911471 100644 --- a/pkg/tsdb/elasticsearch/response_parser_frontend_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_frontend_test.go @@ -1356,53 +1356,6 @@ func TestTwoBucketScripts(t *testing.T) { requireFloatAt(t, 48.0, fields[4], 1) } -func TestRawData(t *testing.T) { - query := []byte(` - [ - { - "refId": "A", - "metrics": [{ "type": "raw_data", "id": "1" }], - "bucketAggs": [] - } - ] - `) - - response := []byte(` - { - "responses": [ - { - "hits": { - "total": { "relation": "eq", "value": 1 }, - "hits": [ - { - "_id": "1", - "_type": "_doc", - "_index": "index", - "_source": { "sourceProp": "asd" } - } - ] - } - } - ] - } - `) - - result, err := queryDataTest(query, response) - require.NoError(t, err) - - require.Len(t, result.response.Responses, 1) - // frames := result.response.Responses["A"].Frames - // require.True(t, len(frames) > 0) // FIXME - - // for _, field := range frames[0].Fields { - // trueValue := true - // filterableConfig := data.FieldConfig{Filterable: &trueValue} - - // // we need to test that the only changed setting is `filterable` - // require.Equal(t, filterableConfig, *field.Config) // FIXME - // } -} - func TestLogsAndCount(t *testing.T) { query := []byte(` [ diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index d2d58315f85..94e3fefb7ea 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -1348,7 +1348,7 @@ func TestResponseParser(t *testing.T) { require.Len(t, dataframes, 1) frame := dataframes[0] - require.Equal(t, 16, len(frame.Fields)) + require.Equal(t, 15, len(frame.Fields)) // Fields have the correct length require.Equal(t, 2, frame.Fields[0].Len()) // First field is timeField @@ -1356,14 +1356,60 @@ func TestResponseParser(t *testing.T) { // Correctly uses string types require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type()) // Correctly detects float64 types - require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[6].Type()) + require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[5].Type()) // Correctly detects json types - require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[7].Type()) + require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[6].Type()) // Correctly flattens fields - require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name) - require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type()) + require.Equal(t, "nested.field.double_nested", frame.Fields[11].Name) + require.Equal(t, data.FieldTypeNullableString, frame.Fields[11].Type()) // Correctly detects type even if first value is null - require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type()) + require.Equal(t, data.FieldTypeNullableString, frame.Fields[14].Type()) + }) + t.Run("Raw data query filterable fields", func(t *testing.T) { + query := []byte(` + [ + { + "refId": "A", + "metrics": [{ "type": "raw_data", "id": "1" }], + "bucketAggs": [] + } + ] + `) + + response := []byte(` + { + "responses": [ + { + "hits": { + "total": { "relation": "eq", "value": 1 }, + "hits": [ + { + "_id": "1", + "_type": "_doc", + "_index": "index", + "_source": { "sourceProp": "asd" } + } + ] + } + } + ] + } + `) + + result, err := queryDataTest(query, response) + require.NoError(t, err) + + require.Len(t, result.response.Responses, 1) + frames := result.response.Responses["A"].Frames + require.True(t, len(frames) > 0) // FIXME + + for _, field := range frames[0].Fields { + trueValue := true + filterableConfig := data.FieldConfig{Filterable: &trueValue} + + // we need to test that the only changed setting is `filterable` + require.Equal(t, filterableConfig, *field.Config) // FIXME + } }) }) diff --git a/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc index 423d4771def..4e4f6774450 100644 --- a/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc +++ b/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc @@ -2,18 +2,18 @@ // // Frame[0] // Name: -// Dimensions: 17 Fields by 5 Rows -// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ -// | Name: @timestamp | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: line | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort | -// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | -// | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | -// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ -// | 2023-02-09T14:40:01.475Z | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:01.475Z","abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","level":"info","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | log text [106619125] | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] | -// | 2023-02-09T14:40:00.513Z | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:00.513Z","abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","level":"error","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}]} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | log text with [781660944] | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] | -// | 2023-02-09T14:39:59.556Z | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:59.556Z","abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","level":"info","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}]} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | log text [894867430] | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] | -// | 2023-02-09T14:39:58.608Z | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:58.608Z","abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","level":"info","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | log text [478598889] | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] | -// | 2023-02-09T14:39:57.665Z | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:57.665Z","abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","level":"info","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | log text [526995818] | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] | -// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// Dimensions: 16 Fields by 5 Rows +// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// | Name: @timestamp | Name: _id | Name: _index | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: line | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort | +// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | +// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// | 2023-02-09T14:40:01.475Z | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | null | null | 81 | 10.911972180833306 | null | true | val3 | info | log text [106619125] | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] | +// | 2023-02-09T14:40:00.513Z | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | null | null | 80 | 62.94120607636795 | null | false | val3 | error | log text with [781660944] | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] | +// | 2023-02-09T14:39:59.556Z | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | null | def | 79 | 53.323706427230455 | null | true | val1 | info | log text [894867430] | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] | +// | 2023-02-09T14:39:58.608Z | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | null | def | 78 | 82.72012623471589 | null | false | val1 | info | log text [478598889] | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] | +// | 2023-02-09T14:39:57.665Z | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | null | def | 77 | 35.05784443331803 | null | false | val3 | info | log text [526995818] | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] | +// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ // // // 🌟 This was machine generated. Do not edit. 🌟 @@ -56,17 +56,6 @@ "filterable": true } }, - { - "name": "_source", - "type": "other", - "typeInfo": { - "frame": "json.RawMessage", - "nullable": true - }, - "config": { - "filterable": true - } - }, { "name": "_type", "type": "other", @@ -235,126 +224,6 @@ "logs-2023.02.09", "logs-2023.02.09" ], - [ - { - "@timestamp": "2023-02-09T14:40:01.475Z", - "abc": null, - "counter": 81, - "float": 10.911972180833306, - "is_true": true, - "label": "val3", - "level": "info", - "line": "log text [106619125]", - "location": "-42.73465234425797, -14.097854057104112", - "nested_field.internal.nested": "value1", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ] - }, - { - "@timestamp": "2023-02-09T14:40:00.513Z", - "abc": null, - "counter": 80, - "float": 62.94120607636795, - "is_true": false, - "label": "val3", - "level": "error", - "line": "log text with [781660944]", - "location": "42.07571917624318, 15.95725088484611", - "nested_field.internal.nested": "value2", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "square" - } - ] - }, - { - "@timestamp": "2023-02-09T14:39:59.556Z", - "abc": "def", - "counter": 79, - "float": 53.323706427230455, - "is_true": true, - "label": "val1", - "level": "info", - "line": "log text [894867430]", - "location": "-38.27341566189766, -23.66739642570781", - "nested_field.internal.nested": "value3", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "square" - } - ] - }, - { - "@timestamp": "2023-02-09T14:39:58.608Z", - "abc": "def", - "counter": 78, - "float": 82.72012623471589, - "is_true": false, - "label": "val1", - "level": "info", - "line": "log text [478598889]", - "location": "12.373240290451287, 43.265493464362024", - "nested_field.internal.nested": "value4", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ] - }, - { - "@timestamp": "2023-02-09T14:39:57.665Z", - "abc": "def", - "counter": 77, - "float": 35.05784443331803, - "is_true": false, - "label": "val3", - "level": "info", - "line": "log text [526995818]", - "location": "-31.524344042228194, -32.11254790120572", - "nested_field.internal.nested": "value5", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ] - } - ], [ null, null,