From 3145660f5a1b13315db92da84245c2d8e9a3c933 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Sat, 6 May 2023 10:00:43 +0200 Subject: [PATCH] Elasticsearch: Run getTerms trough resource call if enableElasticsearchBackendQuerying enabled (#67848) * Elasticsearch: Run getTerms torugh backend if toggle enabled * Add template variables to devenv dashboard for easier testing * Add TODO * Run feature toggle gen to fix build --- .../elasticsearch_complex.json | 508 +++++++++++++++++- pkg/tsdb/elasticsearch/elasticsearch.go | 5 +- .../elasticsearch/datasource.test.ts | 63 +++ .../datasource/elasticsearch/datasource.ts | 23 +- 4 files changed, 591 insertions(+), 8 deletions(-) diff --git a/devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json b/devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json index 5c277b491f2..dc4e4b7810c 100644 --- a/devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json +++ b/devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json @@ -5274,18 +5274,520 @@ ], "title": "Logs", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 97, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "hostname: $hostname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "$hostname count", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "metrics": [ + { + "field": "float", + "id": "1", + "type": "avg" + } + ], + "query": "hostname: $hostname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "$hostname average float", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "hostname: $hostname AND level: $level_by_hostname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "$hostname and $level_by_hostname count", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "metrics": [ + { + "field": "float", + "id": "1", + "type": "avg" + } + ], + "query": "hostname: $hostname AND level: $level_by_hostname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "$hostname and $level_by_hostname average float", + "type": "timeseries" + } + ], + "title": "Using template variables", + "type": "row" } ], "refresh": "", "schemaVersion": 38, "style": "dark", - "tags": [ + "tags": [ "gdev", "elasticsearch", "datasource-test" ], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": true, + "text": "hostname2", + "value": "hostname2" + }, + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "definition": "{\"find\": \"terms\", \"field\": \"hostname\"}", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "hostname", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"hostname\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "error", + "value": "error" + }, + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "definition": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "level_by_hostname", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "gdev-elasticsearch" + }, + "filters": [], + "hide": 0, + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] }, "time": { "from": "now-6h", @@ -5293,7 +5795,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Datasource tests - Elasticsearch complex", + "title": "Datasource tests - Elasticsearch complex with template variables", "uid": "es_complex", "version": 1, "weekStart": "" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 546345d070a..13af0911895 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -183,8 +183,9 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq logger := eslog.FromContext(ctx) // allowed paths for resource calls: // - empty string for fetching db version - // - /?/_mapping for fetching index mapping - if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") { + // - ?/_mapping for fetching index mapping + // - _msearch for executing getTerms queries + if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" { return fmt.Errorf("invalid resource URL: %s", req.Path) } diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index 149103c6ff7..479e9e20852 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -1327,6 +1327,69 @@ describe('ElasticDatasource using backend', () => { }); }); + describe('metricFindQuery', () => { + async function runScenario() { + const data = { + responses: [ + { + aggregations: { + '1': { + buckets: [ + { doc_count: 1, key: 'test' }, + { + doc_count: 2, + key: 'test2', + key_as_string: 'test2_as_string', + }, + ], + }, + }, + }, + ], + }; + + const { ds } = getTestContext(); + const postResourceMock = jest.spyOn(ds, 'postResource'); + postResourceMock.mockResolvedValue(data); + + const results = await ds.metricFindQuery('{"find": "terms", "field": "test"}'); + + expect(ds.postResource).toHaveBeenCalledTimes(1); + const requestOptions = postResourceMock.mock.calls[0][1]; + const parts = requestOptions.split('\n'); + const header = JSON.parse(parts[0]); + const body = JSON.parse(parts[1]); + + return { results, body, header }; + } + + it('should get results', async () => { + const { results } = await runScenario(); + expect(results.length).toEqual(2); + }); + + it('should use key or key_as_string', async () => { + const { results } = await runScenario(); + expect(results[0].text).toEqual('test'); + expect(results[1].text).toEqual('test2_as_string'); + }); + + it('should not set search type to count', async () => { + const { header } = await runScenario(); + expect(header.search_type).not.toEqual('count'); + }); + + it('should set size to 0', async () => { + const { body } = await runScenario(); + expect(body.size).toBe(0); + }); + + it('should not set terms aggregation size to 0', async () => { + const { body } = await runScenario(); + expect(body['aggs']['1']['terms'].size).not.toBe(0); + }); + }); + describe('getFields', () => { const getFieldsMockData = { '[test-]YYYY.MM.DD': { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 6647112e8c9..7e2cad0728e 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -30,7 +30,7 @@ import { LogRowContextOptions, SupplementaryQueryOptions, } from '@grafana/data'; -import { DataSourceWithBackend, getDataSourceSrv, config } from '@grafana/runtime'; +import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime'; import { queryLogsVolume } from 'app/core/logsModel'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; @@ -148,6 +148,18 @@ export class ElasticDatasource this.legacyQueryRunner = new LegacyQueryRunner(this, this.templateSrv); } + getResourceRequest(path: string, params?: BackendSrvRequest['params'], options?: Partial) { + return this.getResource(path, params, options); + } + + postResourceRequest(path: string, data?: BackendSrvRequest['data'], options?: Partial) { + const resourceOptions = options ?? {}; + resourceOptions.headers = resourceOptions.headers ?? {}; + resourceOptions.headers['content-type'] = 'application/x-ndjson'; + + return this.postResource(path, data, resourceOptions); + } + async importFromAbstractQueries(abstractQueries: AbstractQuery[]): Promise { return abstractQueries.map((abstractQuery) => this.languageProvider.importFromAbstractQuery(abstractQuery)); } @@ -532,7 +544,12 @@ export class ElasticDatasource const url = this.getMultiSearchUrl(); - return this.legacyQueryRunner.request('POST', url, esQuery).pipe( + const termsObservable = config.featureToggles.enableElasticsearchBackendQuerying + ? // TODO: This is run trough resource call, but maybe should run trough query + from(this.postResourceRequest(url, esQuery)) + : this.legacyQueryRunner.request('POST', url, esQuery); + + return termsObservable.pipe( map((res) => { if (!res.responses[0].aggregations) { return []; @@ -743,7 +760,7 @@ export class ElasticDatasource private getDatabaseVersionUncached(): Promise { // we want this function to never fail const getDbVersionObservable = config.featureToggles.enableElasticsearchBackendQuerying - ? from(this.getResource('')) + ? from(this.getResourceRequest('')) : this.legacyQueryRunner.request('GET', '/'); return lastValueFrom(getDbVersionObservable).then(