diff --git a/docs/sources/datasources/elasticsearch.md b/docs/sources/datasources/elasticsearch.md index 7cef8d4a46e..356d3dfc850 100644 --- a/docs/sources/datasources/elasticsearch.md +++ b/docs/sources/datasources/elasticsearch.md @@ -80,6 +80,13 @@ number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 se | `s` | second | | `ms` | millisecond | +### X-Pack enabled + +Enables `X-Pack` specific features and options, providing the query editor with additional aggregations such as `Rate` and `Top Metrics`. + +#### Include frozen indices + +When `X-Pack enabled` is active and the configured Elasticsearch version is higher than `6.6.0`, you can configure Grafana to not ignore [frozen indices](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/frozen-indices.html) when performing search requests. ### Logs There are two parameters, `Message field name` and `Level field name`, that can optionally be configured from the data source settings page that determine diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 7798e0034ca..f1b9ad9ea7f 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -334,12 +334,23 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque } func (c *baseClientImpl) getMultiSearchQueryParameters() string { + var qs []string + if c.version.Major() >= 7 { maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5) - return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests) + qs = append(qs, fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)) } - return "" + // Querying frozen indices was added in 6.6 with xpack + includeFrozen := c.getSettings().Get("includeFrozen").MustBool(false) + xpack := c.getSettings().Get("xpack").MustBool(false) + allowedFrozenIndicesVersionRange, _ := semver.NewConstraint(">=6.6.0") + + if (allowedFrozenIndicesVersionRange.Check(c.version)) && includeFrozen && xpack { + qs = append(qs, "ignore_throttled=false") + } + + return strings.Join(qs, "&") } func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index 61537d214a9..64e8312e44d 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -253,10 +253,12 @@ func TestClient_ExecuteMultisearch(t *testing.T) { httpClientScenario(t, "Given a fake http client and a v5.6 client with response", &models.DataSource{ Database: "[metrics-]YYYY.MM.DD", JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 56, + "esVersion": "5.6.0", "maxConcurrentShardRequests": 100, "timeField": "@timestamp", "interval": "Daily", + "includeFrozen": true, + "xpack": true, }), }, func(sc *scenarioContext) { sc.responseBody = `{ @@ -276,6 +278,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) { require.NotNil(t, sc.request) assert.Equal(t, http.MethodPost, sc.request.Method) assert.Equal(t, "/_msearch", sc.request.URL.Path) + assert.NotContains(t, sc.request.URL.RawQuery, "ignore_throttled=") require.NotNil(t, sc.requestBody) @@ -305,10 +308,12 @@ func TestClient_ExecuteMultisearch(t *testing.T) { httpClientScenario(t, "Given a fake http client and a v7.0 client with response", &models.DataSource{ Database: "[metrics-]YYYY.MM.DD", JsonData: simplejson.NewFromAny(map[string]interface{}{ - "esVersion": 70, + "esVersion": "7.0.0", "maxConcurrentShardRequests": 6, "timeField": "@timestamp", "interval": "Daily", + "includeFrozen": true, + "xpack": true, }), }, func(sc *scenarioContext) { sc.responseBody = `{ @@ -328,7 +333,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) { require.NotNil(t, sc.request) assert.Equal(t, http.MethodPost, sc.request.Method) assert.Equal(t, "/_msearch", sc.request.URL.Path) - assert.Equal(t, "max_concurrent_shard_requests=6", sc.request.URL.RawQuery) + assert.Equal(t, "max_concurrent_shard_requests=6&ignore_throttled=false", sc.request.URL.RawQuery) require.NotNil(t, sc.requestBody) @@ -346,6 +351,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) { assert.True(t, jHeader.Get("ignore_unavailable").MustBool(false)) assert.Equal(t, "query_then_fetch", jHeader.Get("search_type").MustString()) assert.Empty(t, jHeader.Get("max_concurrent_shard_requests")) + assert.False(t, jHeader.Get("ignore_throttled").MustBool()) assert.Equal(t, "15000*@hostname", jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString()) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index d932e71dc59..1333ef29a2c 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -67,7 +67,8 @@ func (rp *responseParser) getTimeSeries() (plugins.DataResponse, error) { } queryRes := plugins.DataQueryResult{ - Meta: debugInfo, + Meta: debugInfo, + Dataframes: plugins.NewDecodedDataFrames(data.Frames{}), } props := make(map[string]string) err := rp.processBuckets(res.Aggregations, target, &queryRes, props, 0) diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx index ecaead176a6..9cf6c0233f8 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx @@ -145,17 +145,29 @@ export const ElasticDetails = ({ value, onChange }: Props) => {
+ + {gte(value.jsonData.esVersion, '6.6.0') && value.jsonData.xpack && ( +
+ +
+ )} ); }; +// TODO: Use change handlers from @grafana/data const changeHandler = ( key: keyof DataSourceSettings, value: Props['value'], @@ -167,6 +179,7 @@ const changeHandler = ( }); }; +// TODO: Use change handlers from @grafana/data const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => ( event: React.SyntheticEvent ) => { diff --git a/public/app/plugins/datasource/elasticsearch/configuration/utils.ts b/public/app/plugins/datasource/elasticsearch/configuration/utils.ts index 70c84d4ed0d..d4a4656267d 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/utils.ts +++ b/public/app/plugins/datasource/elasticsearch/configuration/utils.ts @@ -19,6 +19,7 @@ export const coerceOptions = ( options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion), logMessageField: options.jsonData.logMessageField || '', logLevelField: options.jsonData.logLevelField || '', + includeFrozen: options.jsonData.includeFrozen ?? false, }, }; }; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index a70e39998a4..c109c5505d7 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -896,6 +896,42 @@ describe('ElasticDatasource', function (this: any) { }); }); +describe('getMultiSearchUrl', () => { + describe('When esVersion >= 6.6.0', () => { + it('Should add correct params to URL if "includeFrozen" is enabled', () => { + const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: true, xpack: true } }); + + expect(ds.getMultiSearchUrl()).toMatch(/ignore_throttled=false/); + }); + + it('Should NOT add ignore_throttled if "includeFrozen" is disabled', () => { + const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: false, xpack: true } }); + + expect(ds.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + }); + + it('Should NOT add ignore_throttled if "xpack" is disabled', () => { + const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: true, xpack: false } }); + + expect(ds.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + }); + }); + + describe('When esVersion < 6.6.0', () => { + it('Should NOT add ignore_throttled params regardless of includeFrozen', () => { + const { ds: dsWithIncludeFrozen } = getTestContext({ + jsonData: { esVersion: '5.6.0', includeFrozen: false, xpack: true }, + }); + const { ds: dsWithoutIncludeFrozen } = getTestContext({ + jsonData: { esVersion: '5.6.0', includeFrozen: true, xpack: true }, + }); + + expect(dsWithIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + expect(dsWithoutIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + }); + }); +}); + describe('enhanceDataFrame', () => { it('adds links to dataframe', () => { const df = new MutableDataFrame({ diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index e3815bd9e6e..51be789c091 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -73,6 +73,7 @@ export class ElasticDatasource extends DataSourceApi, @@ -99,6 +100,7 @@ export class ElasticDatasource extends DataSourceApi { diff --git a/public/app/plugins/datasource/elasticsearch/types.ts b/public/app/plugins/datasource/elasticsearch/types.ts index 5a5c48bdf61..57903a05057 100644 --- a/public/app/plugins/datasource/elasticsearch/types.ts +++ b/public/app/plugins/datasource/elasticsearch/types.ts @@ -20,6 +20,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData { logMessageField?: string; logLevelField?: string; dataLinks?: DataLinkConfig[]; + includeFrozen?: boolean; } interface MetricConfiguration {