From 9fb8339f8712b9d83c24482ed2ca5f9618035c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Jamr=C3=B3z?= Date: Mon, 17 Jan 2022 15:45:09 +0100 Subject: [PATCH] Elastic: Allow using long/int as date field for alerts (#44027) * Use integers for time range filter Previously it was passed as a string which is automatically converted by Elastic to a number only if the field type is "date". For other types (e.g. "long") such conversion doesn't work. In theory "date" could be passed as a formatted string but we don't use it this way and always pass it as a number so it is safe to always pass numbers, not strings. * Fix time_series_query_test * Retrigger build --- pkg/tsdb/elasticsearch/client/models.go | 8 +++---- .../elasticsearch/client/search_request.go | 2 +- .../client/search_request_test.go | 10 ++++----- pkg/tsdb/elasticsearch/time_series_query.go | 8 +++---- .../elasticsearch/time_series_query_test.go | 21 +++++++++---------- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 611763efc35..ec5d9f68d67 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -136,8 +136,8 @@ func (f *QueryStringFilter) MarshalJSON() ([]byte, error) { type RangeFilter struct { Filter Key string - Gte string - Lte string + Gte int64 + Lte int64 Format string } @@ -264,8 +264,8 @@ type TermsAggregation struct { // ExtendedBounds represents extended bounds type ExtendedBounds struct { - Min string `json:"min"` - Max string `json:"max"` + Min int64 `json:"min"` + Max int64 `json:"max"` } // GeoHashGridAggregation represents a geo hash grid aggregation diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index a35de074c88..c1625adf4e5 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -235,7 +235,7 @@ func (b *FilterQueryBuilder) Build() ([]Filter, error) { } // AddDateRangeFilter adds a new time range filter -func (b *FilterQueryBuilder) AddDateRangeFilter(timeField, lte, gte, format string) *FilterQueryBuilder { +func (b *FilterQueryBuilder) AddDateRangeFilter(timeField string, lte, gte int64, format string) *FilterQueryBuilder { b.filters = append(b.filters, &RangeFilter{ Key: timeField, Lte: lte, diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go index f9c29ac25f3..98f99b42f9a 100644 --- a/pkg/tsdb/elasticsearch/client/search_request_test.go +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -50,7 +50,7 @@ func TestSearchRequest(t *testing.T) { b.Size(200) b.SortDesc(timeField, "boolean") filters := b.Query().Bool().Filter() - filters.AddDateRangeFilter(timeField, "$timeTo", "$timeFrom", DateFormatEpochMS) + filters.AddDateRangeFilter(timeField, 10, 5, DateFormatEpochMS) filters.AddQueryStringFilter("test", true) t.Run("When building search request", func(t *testing.T) { @@ -71,8 +71,8 @@ func TestSearchRequest(t *testing.T) { t.Run("Should have range filter", func(t *testing.T) { f, ok := sr.Query.Bool.Filters[0].(*RangeFilter) require.True(t, ok) - require.Equal(t, "$timeFrom", f.Gte) - require.Equal(t, "$timeTo", f.Lte) + require.Equal(t, int64(5), f.Gte) + require.Equal(t, int64(10), f.Lte) require.Equal(t, "epoch_millis", f.Format) }) @@ -95,8 +95,8 @@ func TestSearchRequest(t *testing.T) { require.Equal(t, "boolean", sort.Get("unmapped_type").MustString()) timeRangeFilter := json.GetPath("query", "bool", "filter").GetIndex(0).Get("range").Get(timeField) - require.Equal(t, "$timeFrom", timeRangeFilter.Get("gte").MustString("")) - require.Equal(t, "$timeTo", timeRangeFilter.Get("lte").MustString("")) + require.Equal(t, int64(5), timeRangeFilter.Get("gte").MustInt64()) + require.Equal(t, int64(10), timeRangeFilter.Get("lte").MustInt64()) require.Equal(t, DateFormatEpochMS, timeRangeFilter.Get("format").MustString("")) queryStringFilter := json.GetPath("query", "bool", "filter").GetIndex(1).Get("query_string") diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 56f2b64f884..18cedfd337b 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -37,8 +37,8 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) { ms := e.client.MultiSearch() - from := fmt.Sprintf("%d", e.dataQueries[0].TimeRange.From.UnixNano()/int64(time.Millisecond)) - to := fmt.Sprintf("%d", e.dataQueries[0].TimeRange.To.UnixNano()/int64(time.Millisecond)) + from := e.dataQueries[0].TimeRange.From.UnixNano() / int64(time.Millisecond) + to := e.dataQueries[0].TimeRange.To.UnixNano() / int64(time.Millisecond) result := backend.QueryDataResponse{ Responses: backend.Responses{}, } @@ -62,7 +62,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) { return rp.getTimeSeries() } -func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to string, +func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64, result backend.QueryDataResponse) error { minInterval, err := e.client.GetMinInterval(q.Interval) if err != nil { @@ -243,7 +243,7 @@ func (bucketAgg BucketAgg) generateSettingsForDSL() map[string]interface{} { return bucketAgg.Settings.MustMap() } -func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo int64) es.AggBuilder { aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 67b6ce47fe0..333bebfb111 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -2,7 +2,6 @@ package elasticsearch import ( "encoding/json" - "fmt" "testing" "time" @@ -17,8 +16,8 @@ import ( func TestExecuteTimeSeriesQuery(t *testing.T) { from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) - fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) - toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + fromMs := from.UnixNano() / int64(time.Millisecond) + toMs := to.UnixNano() / int64(time.Millisecond) t.Run("Test execute time series query", func(t *testing.T) { t.Run("With defaults on es 2", func(t *testing.T) { @@ -32,14 +31,14 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { sr := c.multisearchRequests[0].Requests[0] rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) require.Equal(t, rangeFilter.Key, c.timeField) - require.Equal(t, rangeFilter.Lte, toStr) - require.Equal(t, rangeFilter.Gte, fromStr) + require.Equal(t, rangeFilter.Lte, toMs) + require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, sr.Aggs[0].Key, "2") dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) require.Equal(t, dateHistogramAgg.Field, "@timestamp") - require.Equal(t, dateHistogramAgg.ExtendedBounds.Min, fromStr) - require.Equal(t, dateHistogramAgg.ExtendedBounds.Max, toStr) + require.Equal(t, dateHistogramAgg.ExtendedBounds.Min, fromMs) + require.Equal(t, dateHistogramAgg.ExtendedBounds.Max, toMs) }) t.Run("With defaults on es 5", func(t *testing.T) { @@ -53,8 +52,8 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { sr := c.multisearchRequests[0].Requests[0] require.Equal(t, sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, c.timeField) require.Equal(t, sr.Aggs[0].Key, "2") - require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, fromStr) - require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, toStr) + require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, fromMs) + require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, toMs) }) t.Run("With multiple bucket aggs", func(t *testing.T) { @@ -1061,7 +1060,7 @@ func TestSettingsCasting(t *testing.T) { "field": "@timestamp", "id": "2", "settings": { - "interval": "1d" + "interval": "1d" } } ], @@ -1088,7 +1087,7 @@ func TestSettingsCasting(t *testing.T) { "field": "@timestamp", "id": "2", "settings": { - "interval": "1d" + "interval": "1d" } } ],