Elasticsearch: Remove timeSrv dependency (#29770)

* Elasticsearch: use time range from queries or default one

* Fix tests

* Fix language provider tests

* Elasticsearch: remove timeSrv dependency for logs context (#29841)

* Elasticsearch: remove timeSrv dependency for logs context

* Add tests & slighlty improve getIndexList

Co-authored-by: Elfo404 <gio.ricci@grafana.com>

public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui';
 const { Select, Input, FormField } = LegacyForms;
-import { ElasticsearchOptions } from '../types';
+import { ElasticsearchOptions, Interval } from '../types';
 import { DataSourceSettings, SelectableValue } from '@grafana/data';
 
 const indexPatternTypes = [
@@ -170,7 +170,9 @@ const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['va
   });
 };
 
-const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (option: SelectableValue<string>) => {
+const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (
+  option: SelectableValue<Interval | 'none'>
+) => {
   const { database } = value;
   // If option value is undefined it will send its label instead so we have to convert made up value to undefined here.
   const newInterval = option.value === 'none' ? undefined : option.value;

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -15,7 +15,6 @@ import {
 import _ from 'lodash';
 import { ElasticDatasource, enhanceDataFrame } from './datasource';
 import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
-import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 import { ElasticsearchOptions, ElasticsearchQuery } from './types';
 import { Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations';
@@ -61,8 +60,6 @@ describe('ElasticDatasource', function(this: any) {
     getAdhocFilters: jest.fn(() => []),
   };
 
-  const timeSrv: any = createTimeSrv('now-1h');
-
   interface TestContext {
     ds: ElasticDatasource;
   }
@@ -88,15 +85,8 @@ describe('ElasticDatasource', function(this: any) {
   }
 
   function createDatasource(instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>) {
-    createDatasourceWithTime(instanceSettings, timeSrv as TimeSrv);
-  }
-
-  function createDatasourceWithTime(
-    instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
-    timeSrv: TimeSrv
-  ) {
     instanceSettings.jsonData = instanceSettings.jsonData || ({} as ElasticsearchOptions);
-    ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv);
+    ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv);
   }
 
   describe('When testing datasource with index pattern', () => {
@@ -506,14 +496,11 @@ describe('ElasticDatasource', function(this: any) {
     };
 
     beforeEach(() => {
-      createDatasourceWithTime(
+      createDatasource({
-        {
+        url: ELASTICSEARCH_MOCK_URL,
-          url: ELASTICSEARCH_MOCK_URL,
+        database: '[asd-]YYYY.MM.DD',
-          database: '[asd-]YYYY.MM.DD',
+        jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
-          jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
+      } as DataSourceInstanceSettings<ElasticsearchOptions>);
-        } as DataSourceInstanceSettings<ElasticsearchOptions>,
-        twoWeekTimeSrv
-      );
     });
 
     it('should return fields of the newest available index', async () => {
@@ -534,7 +521,8 @@ describe('ElasticDatasource', function(this: any) {
         return Promise.reject({ status: 404 });
       });
 
-      const fieldObjects = await ctx.ds.getFields();
+      const range = twoWeekTimeSrv.timeRange();
+      const fieldObjects = await ctx.ds.getFields(undefined, range);
 
       const fields = _.map(fieldObjects, 'text');
       expect(fields).toEqual(['@timestamp', 'beat.hostname']);
@@ -545,6 +533,7 @@ describe('ElasticDatasource', function(this: any) {
         .subtract(2, 'day')
         .format('YYYY.MM.DD');
 
+      const range = twoWeekTimeSrv.timeRange();
       datasourceRequestMock.mockImplementation(options => {
         if (options.url === `${ELASTICSEARCH_MOCK_URL}/asd-${twoDaysBefore}/_mapping`) {
           return Promise.resolve(basicResponse);
@@ -554,7 +543,7 @@ describe('ElasticDatasource', function(this: any) {
 
       expect.assertions(2);
       try {
-        await ctx.ds.getFields();
+        await ctx.ds.getFields(undefined, range);
       } catch (e) {
         expect(e).toStrictEqual({ status: 500 });
         expect(datasourceRequestMock).toBeCalledTimes(1);
@@ -562,13 +551,14 @@ describe('ElasticDatasource', function(this: any) {
     });
 
     it('should not retry more than 7 indices', async () => {
+      const range = twoWeekTimeSrv.timeRange();
       datasourceRequestMock.mockImplementation(() => {
         return Promise.reject({ status: 404 });
       });
 
       expect.assertions(2);
       try {
-        await ctx.ds.getFields();
+        await ctx.ds.getFields(undefined, range);
       } catch (e) {
         expect(e).toStrictEqual({ status: 404 });
         expect(datasourceRequestMock).toBeCalledTimes(7);
@@ -840,8 +830,7 @@ describe('ElasticDatasource', function(this: any) {
             timeField: '@time',
           },
         } as DataSourceInstanceSettings<ElasticsearchOptions>,
-        templateSrv as TemplateSrv,
+        templateSrv as TemplateSrv
-        timeSrv as TimeSrv
       );
       (dataSource as any).post = jest.fn(() => Promise.resolve({ responses: [] }));
       dataSource.query(createElasticQuery());

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -12,6 +12,10 @@ import {
   LogRowModel,
   Field,
   MetricFindValue,
+  TimeRange,
+  DefaultTimeRange,
+  DateTime,
+  dateTime,
 } from '@grafana/data';
 import LanguageProvider from './language_provider';
 import { ElasticResponse } from './elastic_response';
@@ -21,7 +25,6 @@ import { toUtc } from '@grafana/data';
 import { defaultBucketAgg, hasMetricOfType } from './query_def';
 import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
 import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
-import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
 import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
 import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
@@ -65,8 +68,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
 
   constructor(
     instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
-    private readonly templateSrv: TemplateSrv = getTemplateSrv(),
+    private readonly templateSrv: TemplateSrv = getTemplateSrv()
-    private readonly timeSrv: TimeSrv = getTimeSrv()
   ) {
     super(instanceSettings);
     this.basicAuth = instanceSettings.basicAuth;
@@ -140,9 +142,8 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
    *
    * @param url the url to query the index on, for example `/_mapping`.
    */
-  private get(url: string) {
+  private get(url: string, range = DefaultTimeRange) {
-    const range = this.timeSrv.timeRange();
+    const indexList = this.indexPattern.getIndexList(range.from, range.to);
-    const indexList = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
     if (_.isArray(indexList) && indexList.length) {
       return this.requestAllIndices(indexList, url).then((results: any) => {
         results.data.$$config = results.config;
@@ -370,7 +371,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     );
   }
 
-  getQueryHeader(searchType: any, timeFrom: any, timeTo: any) {
+  getQueryHeader(searchType: any, timeFrom?: DateTime, timeTo?: DateTime): string {
     const queryHeader: any = {
       search_type: searchType,
       ignore_unavailable: true,
@@ -447,9 +448,13 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
     const sortField = row.dataFrame.fields.find(f => f.name === 'sort');
     const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs];
-    const range = this.timeSrv.timeRange();
+    const sort = options?.direction === 'FORWARD' ? 'asc' : 'desc';
-    const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc';
+
-    const header = this.getQueryHeader('query_then_fetch', range.from, range.to);
+    const header =
+      options?.direction === 'FORWARD'
+        ? this.getQueryHeader('query_then_fetch', dateTime(row.timeEpochMs))
+        : this.getQueryHeader('query_then_fetch', undefined, dateTime(row.timeEpochMs));
+
     const limit = options?.limit ?? 10;
     const esQuery = JSON.stringify({
       size: limit,
@@ -459,8 +464,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
             {
               range: {
                 [this.timeField]: {
-                  gte: range.from.valueOf(),
+                  [options?.direction === 'FORWARD' ? 'gte' : 'lte']: row.timeEpochMs,
-                  lte: range.to.valueOf(),
                   format: 'epoch_millis',
                 },
               },
@@ -468,14 +472,14 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
           ],
         },
       },
-      sort: [{ [this.timeField]: direction }, { _doc: direction }],
+      sort: [{ [this.timeField]: sort }, { _doc: sort }],
       search_after: searchAfter,
     });
     const payload = [header, esQuery].join('\n') + '\n';
     const url = this.getMultiSearchUrl();
     const response = await this.post(url, payload);
     const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }];
-    const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction));
+    const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, sort));
     const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField);
     const dataFrame = _.first(logResponse.data);
     if (!dataFrame) {
@@ -576,9 +580,9 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   }
 
   // TODO: instead of being a string, this could be a custom type representing all the elastic types
-  async getFields(type?: string): Promise<MetricFindValue[]> {
+  async getFields(type?: string, range?: TimeRange): Promise<MetricFindValue[]> {
     const configuredEsVersion = this.esVersion;
-    return this.get('/_mapping').then((result: any) => {
+    return this.get('/_mapping', range).then((result: any) => {
       const typeMap: any = {
         float: 'number',
         double: 'number',
@@ -663,8 +667,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     });
   }
 
-  getTerms(queryDef: any) {
+  getTerms(queryDef: any, range = DefaultTimeRange) {
-    const range = this.timeSrv.timeRange();
     const searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count';
     const header = this.getQueryHeader(searchType, range.from, range.to);
     let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
@@ -698,18 +701,19 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     return '_msearch';
   }
 
-  metricFindQuery(query: string): Promise<MetricFindValue[]> {
+  metricFindQuery(query: string, options?: any): Promise<MetricFindValue[]> {
+    const range = options?.range;
     const parsedQuery = JSON.parse(query);
     if (query) {
       if (parsedQuery.find === 'fields') {
         parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
-        return this.getFields(query);
+        return this.getFields(query, range);
       }
 
       if (parsedQuery.find === 'terms') {
         parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
         parsedQuery.query = this.templateSrv.replace(parsedQuery.query || '*', {}, 'lucene');
-        return this.getTerms(query);
+        return this.getTerms(query, range);
       }
     }
 

public/app/plugins/datasource/elasticsearch/index_pattern.ts

@@ -1,9 +1,18 @@
-import { toUtc, dateTime } from '@grafana/data';
+import { toUtc, dateTime, DateTime, DurationUnit } from '@grafana/data';
+import { Interval } from './types';
 
-const intervalMap: any = {
+type IntervalMap = Record<
+  Interval,
+  {
+    startOf: DurationUnit;
+    amount: DurationUnit;
+  }
+>;
+
+const intervalMap: IntervalMap = {
   Hourly: { startOf: 'hour', amount: 'hours' },
   Daily: { startOf: 'day', amount: 'days' },
-  Weekly: { startOf: 'isoWeek', amount: 'weeks' },
+  Weekly: { startOf: 'week', amount: 'weeks' },
   Monthly: { startOf: 'month', amount: 'months' },
   Yearly: { startOf: 'year', amount: 'years' },
 };
@@ -11,7 +20,7 @@ const intervalMap: any = {
 export class IndexPattern {
   private dateLocale = 'en';
 
-  constructor(private pattern: any, private interval?: string) {}
+  constructor(private pattern: string, private interval?: keyof typeof intervalMap) {}
 
   getIndexForToday() {
     if (this.interval) {
@@ -23,16 +32,21 @@ export class IndexPattern {
     }
   }
 
-  getIndexList(from: any, to: any) {
+  getIndexList(from?: DateTime, to?: DateTime) {
+    // When no `from` or `to` is provided, we request data from 7 subsequent/previous indices
+    // for the provided index pattern.
+    // This is useful when requesting log context where the only time data we have is the log
+    // timestamp.
+    const indexOffset = 7;
     if (!this.interval) {
       return this.pattern;
     }
 
     const intervalInfo = intervalMap[this.interval];
-    const start = dateTime(from)
+    const start = dateTime(from || dateTime(to).add(-indexOffset, intervalInfo.amount))
       .utc()
       .startOf(intervalInfo.startOf);
-    const endEpoch = dateTime(to)
+    const endEpoch = dateTime(to || dateTime(from).add(indexOffset, intervalInfo.amount))
       .utc()
       .startOf(intervalInfo.startOf)
       .valueOf();

public/app/plugins/datasource/elasticsearch/language_provider.test.ts

@@ -1,25 +1,15 @@
 import LanguageProvider from './language_provider';
 import { PromQuery } from '../prometheus/types';
 import { ElasticDatasource } from './datasource';
-import { DataSourceInstanceSettings, dateTime } from '@grafana/data';
+import { DataSourceInstanceSettings } from '@grafana/data';
 import { ElasticsearchOptions } from './types';
 import { TemplateSrv } from '../../../features/templating/template_srv';
-import { TimeSrv } from '../../../features/dashboard/services/TimeSrv';
 
 const templateSrvStub = {
   getAdhocFilters: jest.fn(() => [] as any[]),
   replace: jest.fn((a: string) => a),
 } as any;
 
-const timeSrvStub = {
-  timeRange(): any {
-    return {
-      from: dateTime(1531468681),
-      to: dateTime(1531489712),
-    };
-  },
-} as any;
-
 const dataSource = new ElasticDatasource(
   {
     url: 'http://es.com',
@@ -30,8 +20,7 @@ const dataSource = new ElasticDatasource(
       timeField: '@time',
     },
   } as DataSourceInstanceSettings<ElasticsearchOptions>,
-  templateSrvStub as TemplateSrv,
+  templateSrvStub as TemplateSrv
-  timeSrvStub as TimeSrv
 );
 describe('transform prometheus query to elasticsearch query', () => {
   it('Prometheus query with exact equals labels ( 2 labels ) and metric __name__', () => {

public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts

@@ -1,7 +1,7 @@
 ///<amd-dependency path="test/specs/helpers" name="helpers" />
 
 import { IndexPattern } from '../index_pattern';
-import { toUtc, getLocale, setLocale } from '@grafana/data';
+import { toUtc, getLocale, setLocale, dateTime } from '@grafana/data';
 
 describe('IndexPattern', () => {
   const originalLocale = getLocale();
@@ -31,8 +31,8 @@ describe('IndexPattern', () => {
     describe('no interval', () => {
       test('should return correct index', () => {
         const pattern = new IndexPattern('my-metrics');
-        const from = new Date(2015, 4, 30, 1, 2, 3);
+        const from = dateTime(new Date(2015, 4, 30, 1, 2, 3));
-        const to = new Date(2015, 5, 1, 12, 5, 6);
+        const to = dateTime(new Date(2015, 5, 1, 12, 5, 6));
         expect(pattern.getIndexList(from, to)).toEqual('my-metrics');
       });
     });
@@ -40,8 +40,8 @@ describe('IndexPattern', () => {
     describe('daily', () => {
       test('should return correct index list', () => {
         const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
-        const from = new Date(1432940523000);
+        const from = dateTime(new Date(1432940523000));
-        const to = new Date(1433153106000);
+        const to = dateTime(new Date(1433153106000));
 
         const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
 
@@ -51,8 +51,8 @@ describe('IndexPattern', () => {
       test('should format date using western arabic numerals regardless of locale', () => {
         setLocale('ar_SA'); // saudi-arabic, formatting for YYYY.MM.DD looks like "٢٠٢٠.٠٩.٠٣"
         const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
-        const from = new Date(1432940523000);
+        const from = dateTime(new Date(1432940523000));
-        const to = new Date(1433153106000);
+        const to = dateTime(new Date(1433153106000));
 
         const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
 
@@ -60,4 +60,42 @@ describe('IndexPattern', () => {
       });
     });
   });
+
+  describe('when getting index list from single date', () => {
+    it('Should return index matching the starting time and subsequent ones', () => {
+      const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
+      const from = dateTime(new Date(1432940523000));
+
+      const expected = [
+        'asd-2015.05.29',
+        'asd-2015.05.30',
+        'asd-2015.05.31',
+        'asd-2015.06.01',
+        'asd-2015.06.02',
+        'asd-2015.06.03',
+        'asd-2015.06.04',
+        'asd-2015.06.05',
+      ];
+
+      expect(pattern.getIndexList(from)).toEqual(expected);
+    });
+
+    it('Should return index matching the starting time and previous ones', () => {
+      const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
+      const to = dateTime(new Date(1432940523000));
+
+      const expected = [
+        'asd-2015.05.22',
+        'asd-2015.05.23',
+        'asd-2015.05.24',
+        'asd-2015.05.25',
+        'asd-2015.05.26',
+        'asd-2015.05.27',
+        'asd-2015.05.28',
+        'asd-2015.05.29',
+      ];
+
+      expect(pattern.getIndexList(undefined, to)).toEqual(expected);
+    });
+  });
 });

public/app/plugins/datasource/elasticsearch/types.ts

@@ -8,10 +8,12 @@ import {
   MetricAggregationType,
 } from './components/QueryEditor/MetricAggregationsEditor/aggregations';
 
+export type Interval = 'Hourly' | 'Daily' | 'Weekly' | 'Monthly' | 'Yearly';
+
 export interface ElasticsearchOptions extends DataSourceJsonData {
   timeField: string;
   esVersion: number;
-  interval?: string;
+  interval?: Interval;
   timeInterval: string;
   maxConcurrentShardRequests?: number;
   logMessageField?: string;