public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Elasticsearch: Implement processing of logs query results in backend (#63647)

Browse Source 
* Elasticsearch: Add processing of logs query to backend

* Add and fix tests

* Add snapshot tests

* Fix test in ES client

* Small updates, remove redundant logic

* Refactor setPreferredVisType to improve readability
This commit is contained in:
Ivana Huckova
2023-03-01 05:50:56 -05:00
committed by GitHub
parent 8ac92aab19
commit d258c8ef8a
12 changed files with 1061 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/elasticsearch/response_parser_test.go
+127 -2
View File
@@ -1043,6 +1043,127 @@ func TestResponseParser(t *testing.T) {
require.Nil(t, frame.Fields[1].Config)
})
t.Run("Log query", func(t *testing.T) {
targets := map[string]string{
"A": `{
"metrics": [{ "type": "logs" }]
}`,
}
response := `{
"responses":[
{
"hits":{
"total":{
"value":109,
"relation":"eq"
},
"max_score":null,
"hits":[
{
"_index":"logs-2023.02.08",
"_id":"GB2UMYYBfCQ-FCMjayJa",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:55.830Z",
"line":"log text [479231733]",
"counter":"109",
"float":58.253758485091,
"label":"val1",
"lvl":"info",
"location":"17.089705232090438, 41.62861966340297",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": null
},
"sort":[
1675869055830,
4
]
},
{
"_index":"logs-2023.02.08",
"_id":"Fx2UMYYBfCQ-FCMjZyJ_",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:54.835Z",
"line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]",
"counter":"108",
"float":54.5977098233944,
"label":"val1",
"lvl":"info",
"location":"19.766305918490463, 40.42639175509792",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": "def"
},
"sort":[
1675869054835,
7
]
}
]
},
"status":200
}
]
}`
result, err := parseTestResponse(targets, response)
require.NoError(t, err)
require.Len(t, result.Responses, 1)
queryRes := result.Responses["A"]
require.NotNil(t, queryRes)
dataframes := queryRes.Frames
require.Len(t, dataframes, 1)
frame := dataframes[0]
require.Equal(t, 16, len(frame.Fields))
// Fields have the correct length
require.Equal(t, 2, frame.Fields[0].Len())
// First field is timeField
require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type())
// Second is log line
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
require.Equal(t, "line", frame.Fields[1].Name)
// Correctly renames lvl field to level
require.Equal(t, "level", frame.Fields[10].Name)
// Correctly uses string types
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
// Correctly detects float64 types
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[7].Type())
// Correctly detects json types
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[8].Type())
// Correctly flattens fields
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
// Correctly detects type even if first value is null
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
})
t.Run("Raw data query", func(t *testing.T) {
targets := map[string]string{
"A": `{
@@ -1252,7 +1373,11 @@ func TestResponseParser(t *testing.T) {
func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) {
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
timeField := "@timestamp"
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
timeRange := backend.TimeRange{
From: from,
To: to,
@@ -1280,7 +1405,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac
return nil, err
}
return parseResponse(response.Responses, queries, timeField)
return parseResponse(response.Responses, queries, configuredFields)
}
func TestLabelOrderInFieldName(t *testing.T) {