From e98a8bd11bef2d60e320c3c81a12ba76651d721d Mon Sep 17 00:00:00 2001 From: Giordano Ricci Date: Tue, 11 May 2021 09:44:00 +0100 Subject: [PATCH] Elasticsearch: use semver strings to identify ES version (#33646) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Elasticsearch: use proper semver strings to identify ES version * Update BE & tests * refactor BE tests * refactor isValidOption check * update test * Update pkg/tsdb/elasticsearch/client/client.go Co-authored-by: Piotr Jamróz * Update pkg/tsdb/elasticsearch/client/search_request_test.go Co-authored-by: Piotr Jamróz * Remove leftover FIXME comment * Add new test cases for new version format * Docs: add documentation about version dropdown * Update docs/sources/datasources/elasticsearch.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update docs/sources/datasources/elasticsearch.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update docs/sources/datasources/elasticsearch.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update provisioning documentation Co-authored-by: Piotr Jamróz Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/administration/provisioning.md | 2 +- docs/sources/datasources/elasticsearch.md | 9 +- go.mod | 1 + go.sum | 2 + package.json | 2 + pkg/tsdb/elasticsearch/client/client.go | 75 +++++--- pkg/tsdb/elasticsearch/client/client_test.go | 163 +++++++++++------- .../elasticsearch/client/search_request.go | 24 ++- .../client/search_request_test.go | 10 +- pkg/tsdb/elasticsearch/elasticsearch.go | 1 + .../elasticsearch/time_series_query_test.go | 72 ++++---- .../MetricAggregationsEditor/MetricEditor.tsx | 8 +- .../MetricAggregationsEditor/utils.ts | 12 +- .../configuration/ConfigEditor.test.tsx | 17 +- .../configuration/ConfigEditor.tsx | 31 ++-- .../configuration/ElasticDetails.test.tsx | 22 +-- .../configuration/ElasticDetails.tsx | 27 ++- .../elasticsearch/configuration/mocks.ts | 2 +- .../elasticsearch/configuration/utils.ts | 38 ++++ .../elasticsearch/datasource.test.ts | 2 +- .../datasource/elasticsearch/datasource.ts | 22 +-- .../elasticsearch/language_provider.test.ts | 2 +- .../datasource/elasticsearch/query_builder.ts | 11 +- .../elasticsearch/specs/query_builder.test.ts | 15 +- .../plugins/datasource/elasticsearch/types.ts | 9 +- .../plugins/datasource/elasticsearch/utils.ts | 26 +++ 26 files changed, 363 insertions(+), 242 deletions(-) create mode 100644 public/app/plugins/datasource/elasticsearch/configuration/utils.ts diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 36945536c19..a65976dc3f3 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -154,7 +154,7 @@ Since not all datasources have the same configuration settings we only have the | maxSeries | number | Influxdb | Max number of series/tables that Grafana processes | | httpMethod | string | Prometheus | HTTP Method. 'GET', 'POST', defaults to GET | | customQueryParameters | string | Prometheus | Query parameters to add, as a URL-encoded string. | -| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60/70) | +| esVersion | string | Elasticsearch | Elasticsearch version (E.g. `7.0.0`, `7.6.1`) | | timeField | string | Elasticsearch | Which field that should be used as timestamp | | interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' | | logMessageField | string | Elasticsearch | Which field should be used as the log message | diff --git a/docs/sources/datasources/elasticsearch.md b/docs/sources/datasources/elasticsearch.md index fcc7e6411bb..7e92584961d 100644 --- a/docs/sources/datasources/elasticsearch.md +++ b/docs/sources/datasources/elasticsearch.md @@ -56,9 +56,12 @@ a time pattern for the index name or a wildcard. ### Elasticsearch version -Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences on how queries are composed. -Currently the versions available are `2.x`, `5.x`, `5.6+`, `6.0+` or `7.0+`. The value `5.6+` means version 5.6 or higher, but lower than 6.0. The value `6.0+` means -version 6.0 or higher, but lower than 7.0. Finally, `7.0+` means version 7.0 or higher, but lower than 8.0. +Select the version of your Elasticsearch data source from the version selection dropdown. Different query compositions and functionalities are available in the query editor for different versions. +Available Elasticsearch versions are `2.x`, `5.x`, `5.6+`, `6.0+`, and `7.0+`. Select the option that best matches your data source version. + +Grafana assumes that you are running the lowest possible version for a specified range. This ensures that new features or breaking changes in a future Elasticsearch release will not affect your configuration. + +For example, suppose you are running Elasticsearch `7.6.1` and you selected `7.0+`. If a new feature is made available for Elasticsearch `7.5.0` or newer releases, then a `7.5+` option will be available. However, your configuration will not be affected until you explicitly select the new `7.5+` option in your settings. ### Min time interval diff --git a/go.mod b/go.mod index 12e5cc1cf39..1109bfcb296 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( cloud.google.com/go/storage v1.14.0 cuelang.org/go v0.3.2 github.com/BurntSushi/toml v0.3.1 + github.com/Masterminds/semver v1.5.0 github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/aws/aws-sdk-go v1.38.34 github.com/beevik/etree v1.1.0 diff --git a/go.sum b/go.sum index ab3e3c32e21..a86fdf24025 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnl github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= diff --git a/package.json b/package.json index 1aa170a58b1..14493a155d4 100644 --- a/package.json +++ b/package.json @@ -123,6 +123,7 @@ "@types/redux-logger": "3.0.7", "@types/redux-mock-store": "1.0.2", "@types/reselect": "2.2.0", + "@types/semver": "^6.0.0", "@types/slate": "0.47.1", "@types/slate-plain-serializer": "0.6.1", "@types/slate-react": "0.22.5", @@ -302,6 +303,7 @@ "rst2html": "github:thoward/rst2html#990cb89", "rxjs": "6.6.3", "search-query-parser": "1.5.4", + "semver": "^7.1.3", "slate": "0.47.8", "slate-plain-serializer": "0.7.10", "tether": "1.4.7", diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index ccdd3c7fe8d..1efc017dd73 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/tsdb/interval" @@ -34,7 +35,7 @@ var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) // Client represents a client which can interact with elasticsearch api type Client interface { - GetVersion() int + GetVersion() *semver.Version GetTimeField() string GetMinInterval(queryInterval string) (time.Duration, error) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) @@ -42,9 +43,38 @@ type Client interface { EnableDebug() } +func coerceVersion(v *simplejson.Json) (*semver.Version, error) { + versionString, err := v.String() + + if err != nil { + versionNumber, err := v.Int() + if err != nil { + return nil, err + } + + switch versionNumber { + case 2: + return semver.NewVersion("2.0.0") + case 5: + return semver.NewVersion("5.0.0") + case 56: + return semver.NewVersion("5.6.0") + case 60: + return semver.NewVersion("6.0.0") + case 70: + return semver.NewVersion("7.0.0") + default: + return nil, fmt.Errorf("elasticsearch version=%d is not supported", versionNumber) + } + } + + return semver.NewVersion(versionString) +} + // NewClient creates a new elasticsearch client var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) { - version, err := ds.JsonData.Get("esVersion").Int() + version, err := coerceVersion(ds.JsonData.Get("esVersion")) + if err != nil { return nil, fmt.Errorf("elasticsearch version is required, err=%v", err) } @@ -65,34 +95,29 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugi return nil, err } - clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) + clientLog.Info("Creating new client", "version", version.String(), "timeField", timeField, "indices", strings.Join(indices, ", ")) - switch version { - case 2, 5, 56, 60, 70: - return &baseClientImpl{ - ctx: ctx, - ds: ds, - version: version, - timeField: timeField, - indices: indices, - timeRange: timeRange, - }, nil - } - - return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) + return &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + timeRange: timeRange, + }, nil } type baseClientImpl struct { ctx context.Context ds *models.DataSource - version int + version *semver.Version timeField string indices []string timeRange plugins.DataTimeRange debugEnabled bool } -func (c *baseClientImpl) GetVersion() int { +func (c *baseClientImpl) GetVersion() *semver.Version { return c.version } @@ -297,13 +322,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque interval: searchReq.Interval, } - if c.version == 2 { + if c.version.Major() < 5 { mr.header["search_type"] = "count" - } + } else { + allowedVersionRange, _ := semver.NewConstraint(">=5.6.0, <7.0.0") - if c.version >= 56 && c.version < 70 { - maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) - mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests + if allowedVersionRange.Check(c.version) { + maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests + } } multiRequests = append(multiRequests, &mr) @@ -313,7 +340,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque } func (c *baseClientImpl) getMultiSearchQueryParameters() string { - if c.version >= 70 { + if c.version.Major() >= 7 { maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5) return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests) } diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index 76d0c295152..6a933a4fc4a 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -39,10 +39,104 @@ func TestNewClient(t *testing.T) { require.Error(t, err) }) - t.Run("When unsupported version set should return error", func(t *testing.T) { + t.Run("When using legacy version numbers", func(t *testing.T) { + t.Run("When unsupported version set should return error", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 6, + "timeField": "@timestamp", + }), + } + + _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.Error(t, err) + }) + + t.Run("When version 2 should return v2 client", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, "2.0.0", c.GetVersion().String()) + }) + + t.Run("When version 5 should return v5 client", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, "5.0.0", c.GetVersion().String()) + }) + + t.Run("When version 56 should return v5.6 client", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, "5.6.0", c.GetVersion().String()) + }) + + t.Run("When version 60 should return v6.0 client", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 60, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, "6.0.0", c.GetVersion().String()) + }) + + t.Run("When version 70 should return v7.0 client", func(t *testing.T) { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 70, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, "7.0.0", c.GetVersion().String()) + }) + }) + + t.Run("When version is a valid semver string should create a client", func(t *testing.T) { + version := "7.2.4" ds := &models.DataSource{ JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 6, + "esVersion": version, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + require.NoError(t, err) + assert.Equal(t, version, c.GetVersion().String()) + }) + + t.Run("When version is NOT a valid semver string should return error", func(t *testing.T) { + version := "7.NOT_VALID.4" + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": version, "timeField": "@timestamp", }), } @@ -50,71 +144,6 @@ func TestNewClient(t *testing.T) { _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) require.Error(t, err) }) - - t.Run("When version 2 should return v2 client", func(t *testing.T) { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 2, - "timeField": "@timestamp", - }), - } - - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) - require.NoError(t, err) - assert.Equal(t, 2, c.GetVersion()) - }) - - t.Run("When version 5 should return v5 client", func(t *testing.T) { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 5, - "timeField": "@timestamp", - }), - } - - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) - require.NoError(t, err) - assert.Equal(t, 5, c.GetVersion()) - }) - - t.Run("When version 56 should return v5.6 client", func(t *testing.T) { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 56, - "timeField": "@timestamp", - }), - } - - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) - require.NoError(t, err) - assert.Equal(t, 56, c.GetVersion()) - }) - - t.Run("When version 60 should return v6.0 client", func(t *testing.T) { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 60, - "timeField": "@timestamp", - }), - } - - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) - require.NoError(t, err) - assert.Equal(t, 60, c.GetVersion()) - }) - - t.Run("When version 70 should return v7.0 client", func(t *testing.T) { - ds := &models.DataSource{ - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 70, - "timeField": "@timestamp", - }), - } - - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) - require.NoError(t, err) - assert.Equal(t, 70, c.GetVersion()) - }) } func TestClient_ExecuteMultisearch(t *testing.T) { diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index bedc1fe4640..2c02de113b7 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -3,12 +3,13 @@ package es import ( "strings" + "github.com/Masterminds/semver" "github.com/grafana/grafana/pkg/tsdb/interval" ) // SearchRequestBuilder represents a builder which can build a search request type SearchRequestBuilder struct { - version int + version *semver.Version interval interval.Interval index string size int @@ -19,7 +20,7 @@ type SearchRequestBuilder struct { } // NewSearchRequestBuilder create a new search request builder -func NewSearchRequestBuilder(version int, interval interval.Interval) *SearchRequestBuilder { +func NewSearchRequestBuilder(version *semver.Version, interval interval.Interval) *SearchRequestBuilder { builder := &SearchRequestBuilder{ version: version, interval: interval, @@ -87,18 +88,15 @@ func (b *SearchRequestBuilder) SortDesc(field, unmappedType string) *SearchReque // AddDocValueField adds a doc value field to the search request func (b *SearchRequestBuilder) AddDocValueField(field string) *SearchRequestBuilder { // fields field not supported on version >= 5 - if b.version < 5 { + if b.version.Major() < 5 { b.customProps["fields"] = []string{"*", "_source"} - } - - b.customProps["script_fields"] = make(map[string]interface{}) - - if b.version < 5 { b.customProps["fielddata_fields"] = []string{field} } else { b.customProps["docvalue_fields"] = []string{field} } + b.customProps["script_fields"] = make(map[string]interface{}) + return b } @@ -119,12 +117,12 @@ func (b *SearchRequestBuilder) Agg() AggBuilder { // MultiSearchRequestBuilder represents a builder which can build a multi search request type MultiSearchRequestBuilder struct { - version int + version *semver.Version requestBuilders []*SearchRequestBuilder } // NewMultiSearchRequestBuilder creates a new multi search request builder -func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { +func NewMultiSearchRequestBuilder(version *semver.Version) *MultiSearchRequestBuilder { return &MultiSearchRequestBuilder{ version: version, } @@ -275,10 +273,10 @@ type AggBuilder interface { type aggBuilderImpl struct { AggBuilder aggDefs []*aggDef - version int + version *semver.Version } -func newAggBuilder(version int) *aggBuilderImpl { +func newAggBuilder(version *semver.Version) *aggBuilderImpl { return &aggBuilderImpl{ aggDefs: make([]*aggDef, 0), version: version, @@ -367,7 +365,7 @@ func (b *aggBuilderImpl) Terms(key, field string, fn func(a *TermsAggregation, b fn(innerAgg, builder) } - if b.version >= 60 && len(innerAgg.Order) > 0 { + if b.version.Major() >= 6 && len(innerAgg.Order) > 0 { if orderBy, exists := innerAgg.Order[termsOrderTerm]; exists { innerAgg.Order["_key"] = orderBy delete(innerAgg.Order, termsOrderTerm) diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go index be068d63060..e612fba264e 100644 --- a/pkg/tsdb/elasticsearch/client/search_request_test.go +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/Masterminds/semver" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb/interval" @@ -15,7 +16,8 @@ func TestSearchRequest(t *testing.T) { Convey("Test elasticsearch search request", t, func() { timeField := "@timestamp" Convey("Given new search request builder for es version 5", func() { - b := NewSearchRequestBuilder(5, interval.Interval{Value: 15 * time.Second, Text: "15s"}) + version5, _ := semver.NewVersion("5.0.0") + b := NewSearchRequestBuilder(version5, interval.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When building search request", func() { sr, err := b.Build() @@ -390,7 +392,8 @@ func TestSearchRequest(t *testing.T) { }) Convey("Given new search request builder for es version 2", func() { - b := NewSearchRequestBuilder(2, interval.Interval{Value: 15 * time.Second, Text: "15s"}) + version2, _ := semver.NewVersion("2.0.0") + b := NewSearchRequestBuilder(version2, interval.Interval{Value: 15 * time.Second, Text: "15s"}) Convey("When adding doc value field", func() { b.AddDocValueField(timeField) @@ -446,7 +449,8 @@ func TestSearchRequest(t *testing.T) { func TestMultiSearchRequest(t *testing.T) { Convey("Test elasticsearch multi search request", t, func() { Convey("Given new multi search request builder", func() { - b := NewMultiSearchRequestBuilder(0) + version2, _ := semver.NewVersion("2.0.0") + b := NewMultiSearchRequestBuilder(version2) Convey("When adding one search request", func() { b.Search(interval.Interval{Value: 15 * time.Second, Text: "15s"}) diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 3956263f75f..b55660a8f5d 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -32,6 +32,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, } client, err := es.NewClient(ctx, dsInfo, *tsdbQuery.TimeRange) + if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 976bd3200c5..9088be70613 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/Masterminds/semver" "github.com/grafana/grafana/pkg/plugins" es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/tsdb/interval" @@ -22,7 +23,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { Convey("Test execute time series query", t, func() { Convey("With defaults on es 2", func() { - c := newFakeClient(2) + c := newFakeClient("2.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], @@ -43,7 +44,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With defaults on es 5", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], @@ -58,7 +59,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With multiple bucket aggs", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -80,7 +81,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With select field", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -100,7 +101,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by metric agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -130,7 +131,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by count metric agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -154,7 +155,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by percentiles agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -179,7 +180,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by extended stats agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -204,7 +205,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by term", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -231,7 +232,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With term agg and order by term with es6.x", func() { - c := newFakeClient(60) + c := newFakeClient("6.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -258,7 +259,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With metric percentiles", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -291,7 +292,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With filters aggs on es 2", func() { - c := newFakeClient(2) + c := newFakeClient("2.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -322,7 +323,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With filters aggs on es 5", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -353,7 +354,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With raw document metric", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [], @@ -366,7 +367,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With raw document metric size set", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [], @@ -379,7 +380,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With date histogram agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -405,7 +406,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With histogram agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -432,7 +433,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With geo hash grid agg", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -457,7 +458,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With moving average", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -495,7 +496,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With moving average doc count", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -527,7 +528,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With broken moving average", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -563,7 +564,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With cumulative sum", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -601,7 +602,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With cumulative sum doc count", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -633,7 +634,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With broken cumulative sum", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -669,7 +670,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With derivative", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -698,7 +699,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With derivative doc count", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -727,7 +728,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With serial_diff", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -756,7 +757,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With serial_diff doc count", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -785,7 +786,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With bucket_script", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -822,7 +823,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { }) Convey("With bucket_script doc count", func() { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -862,7 +863,7 @@ func TestSettingsCasting(t *testing.T) { to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) t.Run("Correctly transforms moving_average settings", func(t *testing.T) { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -906,7 +907,7 @@ func TestSettingsCasting(t *testing.T) { }) t.Run("Correctly transforms serial_diff settings", func(t *testing.T) { - c := newFakeClient(5) + c := newFakeClient("5.0.0") _, err := executeTsdbQuery(c, `{ "timeField": "@timestamp", "bucketAggs": [ @@ -935,7 +936,7 @@ func TestSettingsCasting(t *testing.T) { } type fakeClient struct { - version int + version *semver.Version timeField string multiSearchResponse *es.MultiSearchResponse multiSearchError error @@ -943,7 +944,8 @@ type fakeClient struct { multisearchRequests []*es.MultiSearchRequest } -func newFakeClient(version int) *fakeClient { +func newFakeClient(versionString string) *fakeClient { + version, _ := semver.NewVersion(versionString) return &fakeClient{ version: version, timeField: "@timestamp", @@ -954,7 +956,7 @@ func newFakeClient(version int) *fakeClient { func (c *fakeClient) EnableDebug() {} -func (c *fakeClient) GetVersion() int { +func (c *fakeClient) GetVersion() *semver.Version { return c.version } diff --git a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/MetricEditor.tsx b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/MetricEditor.tsx index fcee77abb12..38304ef1179 100644 --- a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/MetricEditor.tsx +++ b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/MetricEditor.tsx @@ -21,6 +21,7 @@ import { MetricAggregationType, } from './aggregations'; import { useFields } from '../../../hooks/useFields'; +import { satisfies } from 'semver'; const toOption = (metric: MetricAggregation) => ({ label: metricAggregationConfig[metric.type].label, @@ -39,7 +40,7 @@ const isBasicAggregation = (metric: MetricAggregation) => !metricAggregationConf const getTypeOptions = ( previousMetrics: MetricAggregation[], - esVersion: number + esVersion: string ): Array> => { // we'll include Pipeline Aggregations only if at least one previous metric is a "Basic" one const includePipelineAggregations = previousMetrics.some(isBasicAggregation); @@ -47,10 +48,7 @@ const getTypeOptions = ( return ( Object.entries(metricAggregationConfig) // Only showing metrics type supported by the configured version of ES - .filter(([_, { minVersion = 0, maxVersion = esVersion }]) => { - // TODO: Double check this - return esVersion >= minVersion && esVersion <= maxVersion; - }) + .filter(([_, { versionRange = '*' }]) => satisfies(esVersion, versionRange)) // Filtering out Pipeline Aggregations if there's no basic metric selected before .filter(([_, config]) => includePipelineAggregations || !config.isPipelineAgg) .map(([key, { label }]) => ({ diff --git a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts index 9864d7b716d..40813b18ea3 100644 --- a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts +++ b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts @@ -112,7 +112,7 @@ export const metricAggregationConfig: MetricsConfiguration = { label: 'Moving Average', requiresField: true, isPipelineAgg: true, - minVersion: 2, + versionRange: '>=2.0.0', supportsMissing: false, supportsMultipleBucketPaths: false, hasSettings: true, @@ -135,14 +135,14 @@ export const metricAggregationConfig: MetricsConfiguration = { supportsMissing: false, hasMeta: false, hasSettings: true, - minVersion: 70, + versionRange: '>=7.0.0', defaults: {}, }, derivative: { label: 'Derivative', requiresField: true, isPipelineAgg: true, - minVersion: 2, + versionRange: '>=2.0.0', supportsMissing: false, supportsMultipleBucketPaths: false, hasSettings: true, @@ -154,7 +154,7 @@ export const metricAggregationConfig: MetricsConfiguration = { label: 'Serial Difference', requiresField: true, isPipelineAgg: true, - minVersion: 2, + versionRange: '>=2.0.0', supportsMissing: false, supportsMultipleBucketPaths: false, hasSettings: true, @@ -170,7 +170,7 @@ export const metricAggregationConfig: MetricsConfiguration = { label: 'Cumulative Sum', requiresField: true, isPipelineAgg: true, - minVersion: 2, + versionRange: '>=2.0.0', supportsMissing: false, supportsMultipleBucketPaths: false, hasSettings: true, @@ -184,7 +184,7 @@ export const metricAggregationConfig: MetricsConfiguration = { isPipelineAgg: true, supportsMissing: false, supportsMultipleBucketPaths: true, - minVersion: 2, + versionRange: '>=2.0.0', hasSettings: true, supportsInlineScript: false, hasMeta: false, diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.test.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.test.tsx index eedbaf1202b..ecf6cc97ea3 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.test.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.test.tsx @@ -31,7 +31,7 @@ describe('ConfigEditor', () => { mount( { - expect(options.jsonData.esVersion).toBe(5); + expect(options.jsonData.esVersion).toBe('5.0.0'); expect(options.jsonData.timeField).toBe('@timestamp'); expect(options.jsonData.maxConcurrentShardRequests).toBe(256); }} @@ -41,17 +41,10 @@ describe('ConfigEditor', () => { }); it('should not apply default if values are set', () => { - expect.assertions(3); + const onChange = jest.fn(); - mount( - { - expect(options.jsonData.esVersion).toBe(70); - expect(options.jsonData.timeField).toBe('@time'); - expect(options.jsonData.maxConcurrentShardRequests).toBe(300); - }} - options={createDefaultConfigOptions()} - /> - ); + mount(); + + expect(onChange).toHaveBeenCalledTimes(0); }); }); diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx index 634f2567463..003f19efed3 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx @@ -2,30 +2,23 @@ import React, { useEffect } from 'react'; import { Alert, DataSourceHttpSettings } from '@grafana/ui'; import { DataSourcePluginOptionsEditorProps } from '@grafana/data'; import { ElasticsearchOptions } from '../types'; -import { defaultMaxConcurrentShardRequests, ElasticDetails } from './ElasticDetails'; +import { ElasticDetails } from './ElasticDetails'; import { LogsConfig } from './LogsConfig'; import { DataLinks } from './DataLinks'; import { config } from 'app/core/config'; +import { coerceOptions, isValidOptions } from './utils'; export type Props = DataSourcePluginOptionsEditorProps; -export const ConfigEditor = (props: Props) => { - const { options, onOptionsChange } = props; - // Apply some defaults on initial render +export const ConfigEditor = (props: Props) => { + const { options: originalOptions, onOptionsChange } = props; + const options = coerceOptions(originalOptions); + useEffect(() => { - const esVersion = options.jsonData.esVersion || 5; - onOptionsChange({ - ...options, - jsonData: { - ...options.jsonData, - timeField: options.jsonData.timeField || '@timestamp', - esVersion, - maxConcurrentShardRequests: - options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion), - logMessageField: options.jsonData.logMessageField || '', - logLevelField: options.jsonData.logLevelField || '', - }, - }); + if (!isValidOptions(originalOptions)) { + onOptionsChange(coerceOptions(originalOptions)); + } + // We can't enforce the eslint rule here because we only want to run this once. // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -39,9 +32,9 @@ export const ConfigEditor = (props: Props) => { )} diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.test.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.test.tsx index 177a71c8fde..fa6044ea076 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.test.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.test.tsx @@ -18,7 +18,7 @@ describe('ElasticDetails', () => { it('should not render "Max concurrent Shard Requests" if version is low', () => { const options = createDefaultConfigOptions(); - options.jsonData.esVersion = 5; + options.jsonData.esVersion = '5.0.0'; const wrapper = mount( {}} value={options} />); expect(wrapper.find('input[aria-label="Max concurrent Shard Requests input"]').length).toBe(0); }); @@ -48,16 +48,16 @@ describe('ElasticDetails', () => { describe('version change', () => { const testCases = [ - { version: 50, expectedMaxConcurrentShardRequests: 256 }, - { version: 50, maxConcurrentShardRequests: 50, expectedMaxConcurrentShardRequests: 50 }, - { version: 56, expectedMaxConcurrentShardRequests: 256 }, - { version: 56, maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 256 }, - { version: 56, maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 256 }, - { version: 56, maxConcurrentShardRequests: 200, expectedMaxConcurrentShardRequests: 200 }, - { version: 70, expectedMaxConcurrentShardRequests: 5 }, - { version: 70, maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 5 }, - { version: 70, maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 5 }, - { version: 70, maxConcurrentShardRequests: 6, expectedMaxConcurrentShardRequests: 6 }, + { version: '5.0.0', expectedMaxConcurrentShardRequests: 256 }, + { version: '5.0.0', maxConcurrentShardRequests: 50, expectedMaxConcurrentShardRequests: 50 }, + { version: '5.6.0', expectedMaxConcurrentShardRequests: 256 }, + { version: '5.6.0', maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 256 }, + { version: '5.6.0', maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 256 }, + { version: '5.6.0', maxConcurrentShardRequests: 200, expectedMaxConcurrentShardRequests: 200 }, + { version: '7.0.0', expectedMaxConcurrentShardRequests: 5 }, + { version: '7.0.0', maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 5 }, + { version: '7.0.0', maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 5 }, + { version: '7.0.0', maxConcurrentShardRequests: 6, expectedMaxConcurrentShardRequests: 6 }, ]; const onChangeMock = jest.fn(); diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx index 49d36e76c3a..bc18cfcd1de 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx @@ -3,6 +3,7 @@ import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui' const { Select, Input, FormField } = LegacyForms; import { ElasticsearchOptions, Interval } from '../types'; import { DataSourceSettings, SelectableValue } from '@grafana/data'; +import { gte, lt } from 'semver'; const indexPatternTypes = [ { label: 'No pattern', value: 'none' }, @@ -14,20 +15,18 @@ const indexPatternTypes = [ ]; const esVersions = [ - { label: '2.x', value: 2 }, - { label: '5.x', value: 5 }, - { label: '5.6+', value: 56 }, - { label: '6.0+', value: 60 }, - { label: '7.0+', value: 70 }, + { label: '2.x', value: '2.0.0' }, + { label: '5.x', value: '5.0.0' }, + { label: '5.6+', value: '5.6.0' }, + { label: '6.0+', value: '6.0.0' }, + { label: '7.0+', value: '7.0.0' }, ]; type Props = { value: DataSourceSettings; onChange: (value: DataSourceSettings) => void; }; -export const ElasticDetails = (props: Props) => { - const { value, onChange } = props; - +export const ElasticDetails = ({ value, onChange }: Props) => { return ( <>

Elasticsearch details

@@ -101,7 +100,7 @@ export const ElasticDetails = (props: Props) => { } /> - {value.jsonData.esVersion >= 56 && ( + {gte(value.jsonData.esVersion, '5.6.0') && (
} }; -function getMaxConcurrenShardRequestOrDefault(maxConcurrentShardRequests: number | undefined, version: number): number { - if (maxConcurrentShardRequests === 5 && version < 70) { +function getMaxConcurrenShardRequestOrDefault(maxConcurrentShardRequests: number | undefined, version: string): number { + if (maxConcurrentShardRequests === 5 && lt(version, '7.0.0')) { return 256; } - if (maxConcurrentShardRequests === 256 && version >= 70) { + if (maxConcurrentShardRequests === 256 && gte(version, '7.0.0')) { return 5; } return maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(version); } -export function defaultMaxConcurrentShardRequests(version: number) { - return version >= 70 ? 5 : 256; +export function defaultMaxConcurrentShardRequests(version: string) { + return gte(version, '7.0.0') ? 5 : 256; } diff --git a/public/app/plugins/datasource/elasticsearch/configuration/mocks.ts b/public/app/plugins/datasource/elasticsearch/configuration/mocks.ts index ea7b451b061..f5c5059319a 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/mocks.ts +++ b/public/app/plugins/datasource/elasticsearch/configuration/mocks.ts @@ -5,7 +5,7 @@ import { createDatasourceSettings } from '../../../../features/datasources/mocks export function createDefaultConfigOptions(): DataSourceSettings { return createDatasourceSettings({ timeField: '@time', - esVersion: 70, + esVersion: '7.0.0', interval: 'Hourly', timeInterval: '10s', maxConcurrentShardRequests: 300, diff --git a/public/app/plugins/datasource/elasticsearch/configuration/utils.ts b/public/app/plugins/datasource/elasticsearch/configuration/utils.ts new file mode 100644 index 00000000000..70c84d4ed0d --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/configuration/utils.ts @@ -0,0 +1,38 @@ +import { DataSourceSettings } from '@grafana/data'; +import { valid } from 'semver'; +import { ElasticsearchOptions } from '../types'; +import { coerceESVersion } from '../utils'; +import { defaultMaxConcurrentShardRequests } from './ElasticDetails'; + +export const coerceOptions = ( + options: DataSourceSettings +): DataSourceSettings => { + const esVersion = coerceESVersion(options.jsonData.esVersion); + + return { + ...options, + jsonData: { + ...options.jsonData, + timeField: options.jsonData.timeField || '@timestamp', + esVersion, + maxConcurrentShardRequests: + options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion), + logMessageField: options.jsonData.logMessageField || '', + logLevelField: options.jsonData.logLevelField || '', + }, + }; +}; + +export const isValidOptions = (options: DataSourceSettings): boolean => { + return ( + // esVersion should be a valid semver string + !!valid(options.jsonData.esVersion) && + // timeField should not be empty or nullish + !!options.jsonData.timeField && + // maxConcurrentShardRequests should be a number AND greater than 0 + !!options.jsonData.maxConcurrentShardRequests && + // message & level fields should be defined + options.jsonData.logMessageField !== undefined && + options.jsonData.logLevelField !== undefined + ); +}; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index f543bd1ed4a..eff2b4cc49c 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -203,7 +203,7 @@ describe('ElasticDatasource', function (this: any) { async function setupDataSource(jsonData?: Partial) { jsonData = { interval: 'Daily', - esVersion: 2, + esVersion: '2.0.0', timeField: '@timestamp', ...(jsonData || {}), }; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index b50195cdcf9..cf9ee04eaa8 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -39,7 +39,8 @@ import { } from './components/QueryEditor/BucketAggregationsEditor/aggregations'; import { generate, Observable, of, throwError } from 'rxjs'; import { catchError, first, map, mergeMap, skipWhile, throwIfEmpty } from 'rxjs/operators'; -import { getScriptValue } from './utils'; +import { coerceESVersion, getScriptValue } from './utils'; +import { gte, lt, satisfies } from 'semver'; // Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields. // custom fields can start with underscores, therefore is not safe to exclude anything that starts with one. @@ -62,7 +63,7 @@ export class ElasticDatasource extends DataSourceApi= 56 && this.esVersion < 70) { + if (satisfies(this.esVersion, '>=5.6.0 <7.0.0')) { queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests; } @@ -484,7 +485,7 @@ export class ElasticDatasource extends DataSourceApi 5; + return gte(this.esVersion, '5.0.0'); } getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => { @@ -588,7 +589,7 @@ export class ElasticDatasource extends DataSourceApi { - const configuredEsVersion = this.esVersion; return this.get('/_mapping', range).pipe( map((result) => { const typeMap: any = { @@ -706,7 +706,7 @@ export class ElasticDatasource extends DataSourceApi { - const searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count'; + const searchType = gte(this.esVersion, '5.0.0') ? 'query_then_fetch' : 'count'; const header = this.getQueryHeader(searchType, range.from, range.to); let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef)); @@ -755,7 +755,7 @@ export class ElasticDatasource extends DataSourceApi= 70 && this.maxConcurrentShardRequests) { + if (gte(this.esVersion, '7.0.0') && this.maxConcurrentShardRequests) { return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`; } diff --git a/public/app/plugins/datasource/elasticsearch/language_provider.test.ts b/public/app/plugins/datasource/elasticsearch/language_provider.test.ts index c8b2c9f0c63..9800ac8d1bb 100644 --- a/public/app/plugins/datasource/elasticsearch/language_provider.test.ts +++ b/public/app/plugins/datasource/elasticsearch/language_provider.test.ts @@ -18,7 +18,7 @@ const dataSource = new ElasticDatasource( database: '[asd-]YYYY.MM.DD', jsonData: { interval: 'Daily', - esVersion: 2, + esVersion: '2.0.0', timeField: '@time', }, } as DataSourceInstanceSettings, diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index 95f91523b1f..1cba16e4133 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -1,3 +1,4 @@ +import { gte, lt } from 'semver'; import { Filters, Histogram, @@ -19,9 +20,9 @@ import { convertOrderByToMetricId, getScriptValue } from './utils'; export class ElasticQueryBuilder { timeField: string; - esVersion: number; + esVersion: string; - constructor(options: { timeField: string; esVersion: number }) { + constructor(options: { timeField: string; esVersion: string }) { this.timeField = options.timeField; this.esVersion = options.esVersion; } @@ -50,7 +51,7 @@ export class ElasticQueryBuilder { if (aggDef.settings.orderBy !== void 0) { queryNode.terms.order = {}; - if (aggDef.settings.orderBy === '_term' && this.esVersion >= 60) { + if (aggDef.settings.orderBy === '_term' && gte(this.esVersion, '6.0.0')) { queryNode.terms.order['_key'] = aggDef.settings.order; } else { queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order; @@ -147,7 +148,7 @@ export class ElasticQueryBuilder { ]; // fields field not supported on ES 5.x - if (this.esVersion < 5) { + if (lt(this.esVersion, '5.0.0')) { query.fields = ['*', '_source']; } @@ -443,7 +444,7 @@ export class ElasticQueryBuilder { switch (orderBy) { case 'key': case 'term': - const keyname = this.esVersion >= 60 ? '_key' : '_term'; + const keyname = gte(this.esVersion, '6.0.0') ? '_key' : '_term'; query.aggs['1'].terms.order[keyname] = order; break; case 'doc_count': diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts index 668515bb906..db75d328279 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts @@ -1,12 +1,13 @@ +import { gte, lt } from 'semver'; import { ElasticQueryBuilder } from '../query_builder'; import { ElasticsearchQuery } from '../types'; describe('ElasticQueryBuilder', () => { - const builder = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 2 }); - const builder5x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 5 }); - const builder56 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 56 }); - const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 60 }); - const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 70 }); + const builder = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '2.0.0' }); + const builder5x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '5.0.0' }); + const builder56 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '5.6.0' }); + const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '6.0.0' }); + const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.0.0' }); const allBuilders = [builder, builder5x, builder56, builder6x, builder7x]; @@ -91,7 +92,7 @@ describe('ElasticQueryBuilder', () => { const query = builder.build(target, 100, '1000'); const firstLevel = query.aggs['2']; - if (builder.esVersion >= 60) { + if (gte(builder.esVersion, '6.0.0')) { expect(firstLevel.terms.order._key).toBe('asc'); } else { expect(firstLevel.terms.order._term).toBe('asc'); @@ -642,7 +643,7 @@ describe('ElasticQueryBuilder', () => { } function checkSort(order: any, expected: string) { - if (builder.esVersion < 60) { + if (lt(builder.esVersion, '6.0.0')) { expect(order._term).toBe(expected); expect(order._key).toBeUndefined(); } else { diff --git a/public/app/plugins/datasource/elasticsearch/types.ts b/public/app/plugins/datasource/elasticsearch/types.ts index 6f9b6337537..34493f2274b 100644 --- a/public/app/plugins/datasource/elasticsearch/types.ts +++ b/public/app/plugins/datasource/elasticsearch/types.ts @@ -12,7 +12,7 @@ export type Interval = 'Hourly' | 'Daily' | 'Weekly' | 'Monthly' | 'Yearly'; export interface ElasticsearchOptions extends DataSourceJsonData { timeField: string; - esVersion: number; + esVersion: string; interval?: Interval; timeInterval: string; maxConcurrentShardRequests?: number; @@ -27,8 +27,11 @@ interface MetricConfiguration { supportsInlineScript: boolean; supportsMissing: boolean; isPipelineAgg: boolean; - minVersion?: number; - maxVersion?: number; + /** + * A valid semver range for which the metric is known to be available. + * If omitted defaults to '*'. + */ + versionRange?: string; supportsMultipleBucketPaths: boolean; isSingleMetric?: boolean; hasSettings: boolean; diff --git a/public/app/plugins/datasource/elasticsearch/utils.ts b/public/app/plugins/datasource/elasticsearch/utils.ts index ad6de6a7249..e7969adb974 100644 --- a/public/app/plugins/datasource/elasticsearch/utils.ts +++ b/public/app/plugins/datasource/elasticsearch/utils.ts @@ -4,6 +4,7 @@ import { MetricAggregationWithInlineScript, } from './components/QueryEditor/MetricAggregationsEditor/aggregations'; import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils'; +import { valid } from 'semver'; export const describeMetric = (metric: MetricAggregation) => { if (!isMetricAggregationWithField(metric)) { @@ -91,3 +92,28 @@ export const convertOrderByToMetricId = (orderBy: string): string | undefined => */ export const getScriptValue = (metric: MetricAggregationWithInlineScript) => (typeof metric.settings?.script === 'object' ? metric.settings?.script?.inline : metric.settings?.script) || ''; + +/** + * Coerces the a version string/number to a valid semver string. + * It takes care of also converting from the legacy format (numeric) to the new one. + * @param version + */ +export const coerceESVersion = (version: string | number): string => { + if (typeof version === 'string') { + return valid(version) || '5.0.0'; + } + + switch (version) { + case 2: + return '2.0.0'; + case 56: + return '5.6.0'; + case 60: + return '6.0.0'; + case 70: + return '7.0.0'; + case 5: + default: + return '5.0.0'; + } +};