From f5ac099907febb5c559f86e5e9960745013652af Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 5 May 2023 11:35:30 +0200 Subject: [PATCH] Elasticsearch: Run requestAllIndices trough resource call if enableElasticsearchBackendQuerying enabled (#67825) * Elasticsearch: Run requestAllIndices trough resource call if enabled * Unlock resource call path * Fix lint --- pkg/tsdb/elasticsearch/elasticsearch.go | 4 +- .../elasticsearch/datasource.test.ts | 174 ++++++++++++++++++ .../datasource/elasticsearch/datasource.ts | 11 +- 3 files changed, 184 insertions(+), 5 deletions(-) diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 0f714996d08..546345d070a 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -11,6 +11,7 @@ import ( "net/url" "path" "strconv" + "strings" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" @@ -182,7 +183,8 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq logger := eslog.FromContext(ctx) // allowed paths for resource calls: // - empty string for fetching db version - if req.Path != "" { + // - /?/_mapping for fetching index mapping + if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") { return fmt.Errorf("invalid resource URL: %s", req.Path) } diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index ab9c07af866..149103c6ff7 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -1326,4 +1326,178 @@ describe('ElasticDatasource using backend', () => { expect(version).toBe(null); }); }); + + describe('getFields', () => { + const getFieldsMockData = { + '[test-]YYYY.MM.DD': { + mappings: { + properties: { + '@timestamp_millis': { + type: 'date', + format: 'epoch_millis', + }, + classification_terms: { + type: 'keyword', + }, + ip_address: { + type: 'ip', + }, + justification_blob: { + properties: { + criterion: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + shallow: { + properties: { + jsi: { + properties: { + sdb: { + properties: { + dsel2: { + properties: { + 'bootlegged-gille': { + properties: { + botness: { + type: 'float', + }, + general_algorithm_score: { + type: 'float', + }, + }, + }, + 'uncombed-boris': { + properties: { + botness: { + type: 'float', + }, + general_algorithm_score: { + type: 'float', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + overall_vote_score: { + type: 'float', + }, + }, + }, + }, + }; + + it('should not retry when ES is down', async () => { + const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD'); + const { ds, timeSrv } = getTestContext({ + from: 'now-2w', + jsonData: { interval: 'Daily' }, + }); + + ds.getResource = jest.fn().mockImplementation((options) => { + if (options.url === `test-${twoDaysBefore}/_mapping`) { + return of({ + data: {}, + }); + } + return throwError({ status: 500 }); + }); + + const range = timeSrv.timeRange(); + await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + expect(received[0]).toStrictEqual({ status: 500 }); + expect(ds.getResource).toBeCalledTimes(1); + }); + }); + + it('should not retry more than 7 indices', async () => { + const { ds, timeSrv } = getTestContext({ + from: 'now-2w', + jsonData: { interval: 'Daily' }, + }); + const range = timeSrv.timeRange(); + + ds.getResource = jest.fn().mockImplementation(() => { + return throwError({ status: 404 }); + }); + + await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + expect(received[0]).toStrictEqual('Could not find an available index for this time range.'); + expect(ds.getResource).toBeCalledTimes(7); + }); + }); + + it('should return nested fields', async () => { + const { ds } = getTestContext({ + from: 'now-2w', + jsonData: { interval: 'Daily' }, + }); + + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields()).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual([ + '@timestamp_millis', + 'classification_terms', + 'ip_address', + 'justification_blob.criterion.keyword', + 'justification_blob.criterion', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score', + 'overall_vote_score', + ]); + }); + }); + it('should return number fields', async () => { + const { ds } = getTestContext({}); + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields(['number'])).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual([ + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score', + 'overall_vote_score', + ]); + }); + }); + + it('should return date fields', async () => { + const { ds } = getTestContext({}); + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields(['date'])).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual(['@timestamp_millis']); + }); + }); + }); }); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 7f2f52da49d..6647112e8c9 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -179,9 +179,12 @@ export class ElasticDatasource }).pipe( mergeMap((index) => { // catch all errors and emit an object with an err property to simplify checks later in the pipeline - return this.legacyQueryRunner - .request('GET', indexUrlList[listLen - index - 1]) - .pipe(catchError((err) => of({ err }))); + const path = indexUrlList[listLen - index - 1]; + const requestObservable = config.featureToggles.enableElasticsearchBackendQuerying + ? from(this.getResource(path)) + : this.legacyQueryRunner.request('GET', path); + + return requestObservable.pipe(catchError((err) => of({ err }))); }), skipWhile((resp) => resp?.err?.status === 404), // skip all requests that fail because missing Elastic index throwIfEmpty(() => 'Could not find an available index for this time range.'), // when i === Math.min(listLen, maxTraversals) generate will complete but without emitting any values which means we didn't find a valid index @@ -462,7 +465,7 @@ export class ElasticDatasource return true; } - // equal query type filter, or via typemap translation + // equal query type filter, or via type map translation return type.includes(obj.type) || type.includes(typeMap[obj.type]); };