public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Elasticsearch: Fix processing of logs with not-recognized time format (#67767)

Browse Source 
* Elasticsearch: Fix parsing of invalid time

* Fix lint

* Add more test data to data.js

* Add tests

* Fix lint

* Update pkg/tsdb/elasticsearch/client/search_request.go
This commit is contained in:
Ivana Huckova
2023-05-04 19:33:00 +02:00
committed by GitHub
parent 49ff85ef54
commit 95cf598423
8 changed files with 142 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc
+16 -16
View File
@@ -15,17 +15,17 @@
// }
// Name:
// Dimensions: 17 Fields by 5 Rows
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:01.475Z"} | null | null | 81 | 10.911972180833306 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
// | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:00.513Z"} | null | null | 80 | 62.94120607636795 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
// | 2023-02-09 14:39:59.556 +0000 UTC | log text [894867430] | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","lvl":"info","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:59.556Z"} | null | def | 79 | 53.323706427230455 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | true | val1 | info | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
// | 2023-02-09 14:39:58.608 +0000 UTC | log text [478598889] | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","lvl":"info","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:58.608Z"} | null | def | 78 | 82.72012623471589 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val1 | info | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
// | 2023-02-09 14:39:57.665 +0000 UTC | log text [526995818] | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","lvl":"info","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:57.665Z"} | null | def | 77 | 35.05784443331803 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val3 | info | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// +-----------------------------------+---------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
// +-----------------------------------+---------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | null | 81 | 10.911972180833306 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
// | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | null | 80 | 62.94120607636795 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
// | 2023-02-09 14:39:59.556 +0000 UTC | log text [894867430] | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","lvl":"info","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | def | 79 | 53.323706427230455 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | true | val1 | info | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
// | 2023-02-09 14:39:58.608 +0000 UTC | log text [478598889] | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","lvl":"info","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | def | 78 | 82.72012623471589 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val1 | info | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
// | 2023-02-09 14:39:57.665 +0000 UTC | log text [526995818] | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","lvl":"info","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | def | 77 | 35.05784443331803 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | false | val3 | info | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
// +-----------------------------------+---------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
@@ -292,7 +292,7 @@
"type": "square"
}
],
"testtime": "2023-02-09T14:40:01.475Z"
"testtime": "09/02/2023"
},
{
"abc": null,
@@ -312,7 +312,7 @@
"type": "square"
}
],
"testtime": "2023-02-09T14:40:00.513Z"
"testtime": "09/02/2023"
},
{
"abc": "def",
@@ -332,7 +332,7 @@
"type": "square"
}
],
"testtime": "2023-02-09T14:39:59.556Z"
"testtime": "09/02/2023"
},
{
"abc": "def",
@@ -358,7 +358,7 @@
"type": "square"
}
],
"testtime": "2023-02-09T14:39:58.608Z"
"testtime": "09/02/2023"
},
{
"abc": "def",
@@ -384,7 +384,7 @@
"type": "square"
}
],
"testtime": "2023-02-09T14:39:57.665Z"
"testtime": "09/02/2023"
}
],
[
pkg/tsdb/elasticsearch/testdata_response/logs.response.json
+30 -5
View File
@@ -28,7 +28,7 @@
"_source": {
"abc": null,
"is_true": true,
"testtime": "2023-02-09T14:40:01.475Z",
"testtime": "09/02/2023",
"line": "log text [106619125]",
"counter": 81,
"float": 10.911972180833306,
@@ -55,6 +55,11 @@
}
}
},
"fields": {
"testtime": [
"2023-02-09T14:40:01.475Z"
]
},
"sort": [
1675953601475,
4
@@ -71,7 +76,7 @@
"_source": {
"abc": null,
"is_true": false,
"testtime": "2023-02-09T14:40:00.513Z",
"testtime": "09/02/2023",
"line": "log text with [781660944]",
"counter": 80,
"float": 62.94120607636795,
@@ -92,6 +97,11 @@
}
}
},
"fields": {
"testtime": [
"2023-02-09T14:40:00.513Z"
]
},
"sort": [
1675953600513,
7
@@ -108,7 +118,7 @@
"_source": {
"abc": "def",
"is_true": true,
"testtime": "2023-02-09T14:39:59.556Z",
"testtime": "09/02/2023",
"line": "log text [894867430]",
"counter": 79,
"float": 53.323706427230455,
@@ -129,6 +139,11 @@
}
}
},
"fields": {
"testtime": [
"2023-02-09T14:39:59.556Z"
]
},
"sort": [
1675953599556,
10
@@ -145,7 +160,7 @@
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:58.608Z",
"testtime": "09/02/2023",
"line": "log text [478598889]",
"counter": 78,
"float": 82.72012623471589,
@@ -172,6 +187,11 @@
}
}
},
"fields": {
"testtime": [
"2023-02-09T14:39:58.608Z"
]
},
"sort": [
1675953598608,
15
@@ -188,7 +208,7 @@
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:57.665Z",
"testtime": "09/02/2023",
"line": "log text [526995818]",
"counter": 77,
"float": 35.05784443331803,
@@ -215,6 +235,11 @@
}
}
},
"fields": {
"testtime": [
"2023-02-09T14:39:57.665Z"
]
},
"sort": [
1675953597665,
20