public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Elasticsearch: decouple from timeSrv and templateSrv (#76894)

Browse Source 
* Elasticsearch: remove timeSrv dependencies

* Elastisearch: decouple from templateSrv

* Prettier

* Elasticsearch: improve typescript in mocks file

* Prettier
This commit is contained in:
Matias Chomicki
2023-10-24 10:25:23 +02:00
committed by GitHub
parent afa697f954
commit 4ea4156b08
4 changed files with 98 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/app/plugins/datasource/elasticsearch/LanguageProvider.test.ts
+1 -8
View File
@@ -1,7 +1,5 @@
import { AbstractLabelOperator, AbstractQuery } from '@grafana/data';
import { TemplateSrv } from '../../../features/templating/template_srv';
import LanguageProvider from './LanguageProvider';
import { ElasticDatasource } from './datasource';
import { createElasticDatasource } from './mocks';
@@ -14,12 +12,7 @@ const baseLogsQuery: Partial<ElasticsearchQuery> = {
describe('transform abstract query to elasticsearch query', () => {
let datasource: ElasticDatasource;
beforeEach(() => {
const templateSrvStub = {
getAdhocFilters: jest.fn(() => []),
replace: jest.fn((a: string) => a),
} as unknown as TemplateSrv;
datasource = createElasticDatasource({}, templateSrvStub);
datasource = createElasticDatasource();
});
it('With some labels', () => {
public/app/plugins/datasource/elasticsearch/datasource.test.ts
+56 -128
View File
@@ -8,21 +8,17 @@ import {
DataQueryRequest,
DataQueryResponse,
DataSourceInstanceSettings,
dateMath,
DateTime,
dateTime,
Field,
FieldType,
MutableDataFrame,
RawTimeRange,
SupplementaryQueryType,
TimeRange,
toUtc,
} from '@grafana/data';
import { BackendSrvRequest, FetchResponse, reportInteraction, config } from '@grafana/runtime';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { createFetchResponse } from '../../../../test/helpers/createFetchResponse';
@@ -48,8 +44,8 @@ jest.mock('@grafana/runtime', () => ({
},
}));
const TIMESRV_START = [2022, 8, 21, 6, 10, 10];
const TIMESRV_END = [2022, 8, 24, 6, 10, 21];
const TIME_START = [2022, 8, 21, 6, 10, 10];
const TIME_END = [2022, 8, 24, 6, 10, 21];
const DATAQUERY_BASE = {
requestId: '1',
interval: '',
@@ -62,13 +58,6 @@ const DATAQUERY_BASE = {
startTime: 0,
};
jest.mock('app/features/dashboard/services/TimeSrv', () => ({
...jest.requireActual('app/features/dashboard/services/TimeSrv'),
getTimeSrv: () => ({
timeRange: () => createTimeRange(toUtc(TIMESRV_START), toUtc(TIMESRV_END)),
}),
}));
const createTimeRange = (from: DateTime, to: DateTime): TimeRange => ({
from,
to,
@@ -80,60 +69,29 @@ const createTimeRange = (from: DateTime, to: DateTime): TimeRange => ({
interface TestContext {
data?: Data;
from?: string;
jsonData?: Partial<ElasticsearchOptions>;
database?: string;
fetchMockImplementation?: (options: BackendSrvRequest) => Observable<FetchResponse>;
templateSrvMock?: TemplateSrv;
}
interface Data {
[key: string]: undefined | string | string[] | number | Data | Data[];
}
function getTestContext({
data = { responses: [] },
from = 'now-5m',
jsonData,
fetchMockImplementation,
templateSrvMock,
}: TestContext = {}) {
function getTestContext({ data = { responses: [] }, jsonData, fetchMockImplementation }: TestContext = {}) {
const defaultMock = (options: BackendSrvRequest) => of(createFetchResponse(data));
const fetchMock = jest.spyOn(backendSrv, 'fetch');
fetchMock.mockImplementation(fetchMockImplementation ?? defaultMock);
const timeSrv = {
time: { from, to: 'now' },
timeRange: () => ({
from: dateMath.parse(timeSrv.time.from, false),
to: dateMath.parse(timeSrv.time.to, true),
}),
setTime: (time: RawTimeRange) => {
timeSrv.time = time;
},
} as TimeSrv;
const settings: Partial<DataSourceInstanceSettings<ElasticsearchOptions>> = { url: ELASTICSEARCH_MOCK_URL };
settings.jsonData = jsonData as ElasticsearchOptions;
const templateSrv =
templateSrvMock ??
({
replace: (text?: string) => {
if (text?.startsWith('$')) {
return `resolvedVariable`;
} else {
return text;
}
},
containsTemplate: jest.fn().mockImplementation((text?: string) => text?.includes('$') ?? false),
getAdhocFilters: jest.fn().mockReturnValue([]),
} as unknown as TemplateSrv);
const ds = createElasticDatasource(settings);
const ds = createElasticDatasource(settings, templateSrv);
const timeRange = createTimeRange(toUtc(TIME_START), toUtc(TIME_END));
return { timeSrv, ds, fetchMock, templateSrv };
return { ds, fetchMock, timeRange };
}
describe('ElasticDatasource', () => {
@@ -167,12 +125,12 @@ describe('ElasticDatasource', () => {
},
],
};
const { ds, fetchMock } = getTestContext({
const { ds, fetchMock, timeRange } = getTestContext({
data,
jsonData: { interval: 'Daily', timeField: '@timestamp' },
});
ds.getTagValues({ key: 'test' });
ds.getTagValues({ key: 'test', timeRange: timeRange, filters: [] });
expect(fetchMock).toHaveBeenCalledTimes(1);
const obj = JSON.parse(fetchMock.mock.calls[0][0].data.split('\n')[1]);
@@ -501,7 +459,6 @@ describe('ElasticDatasource', () => {
const { ds } = getTestContext({
fetchMockImplementation: () => throwError(response),
from: undefined,
});
const errObject = {
@@ -566,8 +523,7 @@ describe('ElasticDatasource', () => {
it('should not retry when ES is down', async () => {
const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD');
const { ds, timeSrv, fetchMock } = getTestContext({
from: 'now-2w',
const { ds, fetchMock, timeRange } = getTestContext({
jsonData: { interval: 'Daily' },
fetchMockImplementation: (options) => {
if (options.url === `${ELASTICSEARCH_MOCK_URL}/asd-${twoDaysBefore}/_mapping`) {
@@ -577,9 +533,7 @@ describe('ElasticDatasource', () => {
},
});
const range = timeSrv.timeRange();
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual({ status: 500 });
expect(fetchMock).toBeCalledTimes(1);
@@ -587,16 +541,16 @@ describe('ElasticDatasource', () => {
});
it('should not retry more than 7 indices', async () => {
const { ds, timeSrv, fetchMock } = getTestContext({
from: 'now-2w',
const { ds, fetchMock } = getTestContext({
jsonData: { interval: 'Daily' },
fetchMockImplementation: (options) => {
return throwError({ status: 404 });
},
});
const range = timeSrv.timeRange();
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
const timeRange = createTimeRange(dateTime().subtract(2, 'week'), dateTime());
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual('Could not find an available index for this time range.');
expect(fetchMock).toBeCalledTimes(7);
@@ -893,11 +847,8 @@ describe('ElasticDatasource', () => {
});
it('should correctly add ad hoc filters when interpolating variables in query', () => {
const templateSrvMock = {
replace: (text?: string) => text,
getAdhocFilters: () => [{ key: 'bar', operator: '=', value: 'test' }],
} as unknown as TemplateSrv;
const { ds } = getTestContext({ templateSrvMock });
const adHocFilters = [{ key: 'bar', operator: '=', value: 'test' }];
const { ds } = getTestContext();
const query: ElasticsearchQuery = {
refId: 'A',
bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }],
@@ -905,7 +856,7 @@ describe('ElasticDatasource', () => {
query: 'foo:"bar"',
};
const interpolatedQuery = ds.interpolateVariablesInQueries([query], {})[0];
const interpolatedQuery = ds.interpolateVariablesInQueries([query], {}, adHocFilters)[0];
expect(interpolatedQuery.query).toBe('foo:"bar" AND bar:"test"');
});
@@ -1307,73 +1258,59 @@ describe('queryHasFilter()', () => {
describe('addAdhocFilters', () => {
describe('with invalid filters', () => {
let ds: ElasticDatasource, templateSrv: TemplateSrv;
let ds: ElasticDatasource;
beforeEach(() => {
const context = getTestContext();
ds = context.ds;
templateSrv = context.templateSrv;
});
it('should filter out ad hoc filter without key', () => {
jest.mocked(templateSrv.getAdhocFilters).mockReturnValue([{ key: '', operator: '=', value: 'a', condition: '' }]);
const query = ds.addAdHocFilters('foo:"bar"');
const query = ds.addAdHocFilters('foo:"bar"', [{ key: '', operator: '=', value: 'a', condition: '' }]);
expect(query).toBe('foo:"bar"');
});
it('should filter out ad hoc filter without value', () => {
jest.mocked(templateSrv.getAdhocFilters).mockReturnValue([{ key: 'a', operator: '=', value: '', condition: '' }]);
const query = ds.addAdHocFilters('foo:"bar"');
const query = ds.addAdHocFilters('foo:"bar"', [{ key: 'a', operator: '=', value: '', condition: '' }]);
expect(query).toBe('foo:"bar"');
});
it('should filter out filter ad hoc filter with invalid operator', () => {
jest.mocked(templateSrv.getAdhocFilters).mockReturnValue([{ key: 'a', operator: 'A', value: '', condition: '' }]);
const query = ds.addAdHocFilters('foo:"bar"');
const query = ds.addAdHocFilters('foo:"bar"', [{ key: 'a', operator: 'A', value: '', condition: '' }]);
expect(query).toBe('foo:"bar"');
});
});
describe('with 1 ad hoc filter', () => {
let ds: ElasticDatasource, templateSrvMock: TemplateSrv;
let ds: ElasticDatasource;
beforeEach(() => {
const { ds: datasource, templateSrv } = getTestContext();
const { ds: datasource } = getTestContext();
ds = datasource;
templateSrvMock = templateSrv;
jest
.mocked(templateSrv.getAdhocFilters)
.mockReturnValue([{ key: 'test', operator: '=', value: 'test1', condition: '' }]);
});
it('should correctly add 1 ad hoc filter when query is not empty', () => {
const query = ds.addAdHocFilters('foo:"bar"');
const filters = [{ key: 'test', operator: '=', value: 'test1', condition: '' }];
const query = ds.addAdHocFilters('foo:"bar"', filters);
expect(query).toBe('foo:"bar" AND test:"test1"');
});
it('should correctly add 1 ad hoc filter when query is empty', () => {
expect(ds.addAdHocFilters('')).toBe('test:"test1"');
expect(ds.addAdHocFilters(' ')).toBe('test:"test1"');
expect(ds.addAdHocFilters(' ')).toBe('test:"test1"');
const filters = [{ key: 'test', operator: '=', value: 'test1', condition: '' }];
expect(ds.addAdHocFilters('', filters)).toBe('test:"test1"');
expect(ds.addAdHocFilters(' ', filters)).toBe('test:"test1"');
expect(ds.addAdHocFilters(' ', filters)).toBe('test:"test1"');
});
it('should not fail if the filter value is a number', () => {
jest
.mocked(templateSrvMock.getAdhocFilters)
// @ts-expect-error
.mockReturnValue([{ key: 'key', operator: '=', value: 1, condition: '' }]);
expect(ds.addAdHocFilters('')).toBe('key:"1"');
// @ts-expect-error
expect(ds.addAdHocFilters('', [{ key: 'key', operator: '=', value: 1, condition: '' }])).toBe('key:"1"');
});
it.each(['=', '!=', '=~', '!~', '>', '<', '', ''])(
`should properly build queries with '%s' filters`,
(operator: string) => {
jest
.mocked(templateSrvMock.getAdhocFilters)
.mockReturnValue([{ key: 'key', operator, value: 'value', condition: '' }]);
const filters = [{ key: 'key', operator, value: 'value', condition: '' }];
const query = ds.addAdHocFilters('foo:"bar"', filters);
const query = ds.addAdHocFilters('foo:"bar"');
switch (operator) {
case '=':
expect(query).toBe('foo:"bar" AND key:"value"');
@@ -1398,46 +1335,40 @@ describe('addAdhocFilters', () => {
);
it('should escape characters in filter keys', () => {
jest
.mocked(templateSrvMock.getAdhocFilters)
.mockReturnValue([{ key: 'field:name', operator: '=', value: 'field:value', condition: '' }]);
const query = ds.addAdHocFilters('');
const filters = [{ key: 'field:name', operator: '=', value: 'field:value', condition: '' }];
const query = ds.addAdHocFilters('', filters);
expect(query).toBe('field\\:name:"field:value"');
});
it('should escape characters in filter values', () => {
jest
.mocked(templateSrvMock.getAdhocFilters)
.mockReturnValue([{ key: 'field:name', operator: '=', value: 'field "value"', condition: '' }]);
const query = ds.addAdHocFilters('');
const filters = [{ key: 'field:name', operator: '=', value: 'field "value"', condition: '' }];
const query = ds.addAdHocFilters('', filters);
expect(query).toBe('field\\:name:"field \\"value\\""');
});
});
describe('with multiple ad hoc filters', () => {
let ds: ElasticDatasource;
const filters = [
{ key: 'bar', operator: '=', value: 'baz', condition: '' },
{ key: 'job', operator: '!=', value: 'grafana', condition: '' },
{ key: 'service', operator: '=~', value: 'service', condition: '' },
{ key: 'count', operator: '>', value: '1', condition: '' },
];
beforeEach(() => {
const { ds: datasource, templateSrv } = getTestContext();
const { ds: datasource } = getTestContext();
ds = datasource;
jest.mocked(templateSrv.getAdhocFilters).mockReturnValue([
{ key: 'bar', operator: '=', value: 'baz', condition: '' },
{ key: 'job', operator: '!=', value: 'grafana', condition: '' },
{ key: 'service', operator: '=~', value: 'service', condition: '' },
{ key: 'count', operator: '>', value: '1', condition: '' },
]);
});
it('should correctly add ad hoc filters when query is not empty', () => {
const query = ds.addAdHocFilters('foo:"bar" AND test:"test1"');
const query = ds.addAdHocFilters('foo:"bar" AND test:"test1"', filters);
expect(query).toBe(
'foo:"bar" AND test:"test1" AND bar:"baz" AND -job:"grafana" AND service:/service/ AND count:>1'
);
});
it('should correctly add ad hoc filters when query is empty', () => {
const query = ds.addAdHocFilters('');
const query = ds.addAdHocFilters('', filters);
expect(query).toBe('bar:"baz" AND -job:"grafana" AND service:/service/ AND count:>1');
});
});
@@ -1591,7 +1522,7 @@ describe('ElasticDatasource using backend', () => {
describe('annotationQuery', () => {
describe('results processing', () => {
it('should return simple annotations using defaults', async () => {
const { ds, timeSrv } = getTestContext();
const { ds, timeRange } = getTestContext();
ds.postResourceRequest = jest.fn().mockResolvedValue({
responses: [
{
@@ -1607,7 +1538,7 @@ describe('ElasticDatasource using backend', () => {
const annotations = await ds.annotationQuery({
annotation: {},
range: timeSrv.timeRange(),
range: timeRange,
});
expect(annotations).toHaveLength(2);
@@ -1616,7 +1547,7 @@ describe('ElasticDatasource using backend', () => {
});
it('should return annotation events using options', async () => {
const { ds, timeSrv } = getTestContext();
const { ds, timeRange } = getTestContext();
ds.postResourceRequest = jest.fn().mockResolvedValue({
responses: [
{
@@ -1638,7 +1569,7 @@ describe('ElasticDatasource using backend', () => {
tagsField: '@test_tags',
textField: 'text',
},
range: timeSrv.timeRange(),
range: timeRange,
});
expect(annotations).toHaveLength(2);
expect(annotations[0].time).toBe(1);
@@ -1877,8 +1808,7 @@ describe('ElasticDatasource using backend', () => {
it('should not retry when ES is down', async () => {
const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD');
const { ds, timeSrv } = getTestContext({
from: 'now-2w',
const { ds, timeRange } = getTestContext({
jsonData: { interval: 'Daily' },
});
@@ -1891,8 +1821,7 @@ describe('ElasticDatasource using backend', () => {
return throwError({ status: 500 });
});
const range = timeSrv.timeRange();
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual({ status: 500 });
expect(ds.getResource).toBeCalledTimes(1);
@@ -1900,17 +1829,17 @@ describe('ElasticDatasource using backend', () => {
});
it('should not retry more than 7 indices', async () => {
const { ds, timeSrv } = getTestContext({
from: 'now-2w',
const { ds } = getTestContext({
jsonData: { interval: 'Daily' },
});
const range = timeSrv.timeRange();
ds.getResource = jest.fn().mockImplementation(() => {
return throwError({ status: 404 });
});
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
const timeRange = createTimeRange(dateTime().subtract(2, 'week'), dateTime());
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual('Could not find an available index for this time range.');
expect(ds.getResource).toBeCalledTimes(7);
@@ -1919,7 +1848,6 @@ describe('ElasticDatasource using backend', () => {
it('should return nested fields', async () => {
const { ds } = getTestContext({
from: 'now-2w',
jsonData: { interval: 'Daily' },
});
public/app/plugins/datasource/elasticsearch/datasource.ts
+26 -15
View File
@@ -35,10 +35,17 @@ import {
DataSourceWithToggleableQueryFiltersSupport,
QueryFilterOptions,
ToggleFilterAction,
DataSourceGetTagValuesOptions,
AdHocVariableFilter,
} from '@grafana/data';
import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import {
DataSourceWithBackend,
getDataSourceSrv,
config,
BackendSrvRequest,
TemplateSrv,
getTemplateSrv,
} from '@grafana/runtime';
import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel';
import { getLogLevelFromKey } from '../../../features/logs/utils';
@@ -114,7 +121,6 @@ export class ElasticDatasource
languageProvider: LanguageProvider;
includeFrozen: boolean;
isProxyAccess: boolean;
timeSrv: TimeSrv;
databaseVersion: SemVer | null;
legacyQueryRunner: LegacyQueryRunner;
@@ -157,7 +163,6 @@ export class ElasticDatasource
this.logLevelField = undefined;
}
this.languageProvider = new LanguageProvider(this);
this.timeSrv = getTimeSrv();
this.legacyQueryRunner = new LegacyQueryRunner(this, this.templateSrv);
}
@@ -399,8 +404,12 @@ export class ElasticDatasource
return this.templateSrv.replace(queryString, scopedVars, 'lucene');
}
interpolateVariablesInQueries(queries: ElasticsearchQuery[], scopedVars: ScopedVars | {}): ElasticsearchQuery[] {
return queries.map((q) => this.applyTemplateVariables(q, scopedVars));
interpolateVariablesInQueries(
queries: ElasticsearchQuery[],
scopedVars: ScopedVars,
filters?: AdHocVariableFilter[]
): ElasticsearchQuery[] {
return queries.map((q) => this.applyTemplateVariables(q, scopedVars, filters));
}
async testDatasource() {
@@ -832,9 +841,8 @@ export class ElasticDatasource
return lastValueFrom(this.getFields());
}
getTagValues(options: { key: string }) {
const range = this.timeSrv.timeRange();
return lastValueFrom(this.getTerms({ field: options.key }, range));
getTagValues(options: DataSourceGetTagValuesOptions) {
return lastValueFrom(this.getTerms({ field: options.key }, options.timeRange));
}
targetContainsTemplate(target: ElasticsearchQuery) {
@@ -944,9 +952,8 @@ export class ElasticDatasource
return { ...query, query: expression };
}
addAdHocFilters(query: string) {
const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
if (adhocFilters.length === 0) {
addAdHocFilters(query: string, adhocFilters?: AdHocVariableFilter[]) {
if (!adhocFilters) {
return query;
}
let finalQuery = query;
@@ -958,7 +965,11 @@ export class ElasticDatasource
}
// Used when running queries through backend
applyTemplateVariables(query: ElasticsearchQuery, scopedVars: ScopedVars): ElasticsearchQuery {
applyTemplateVariables(
query: ElasticsearchQuery,
scopedVars: ScopedVars,
filters?: AdHocVariableFilter[]
): ElasticsearchQuery {
// We need a separate interpolation format for lucene queries, therefore we first interpolate any
// lucene query string and then everything else
const interpolateBucketAgg = (bucketAgg: BucketAggregation): BucketAggregation => {
@@ -981,7 +992,7 @@ export class ElasticDatasource
const expandedQuery = {
...query,
datasource: this.getRef(),
query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars)),
query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars), filters),
bucketAggs: query.bucketAggs?.map(interpolateBucketAgg),
};
public/app/plugins/datasource/elasticsearch/mocks.ts
+15 -5
View File
@@ -1,13 +1,10 @@
import { DataSourceInstanceSettings, PluginType } from '@grafana/data';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { TemplateSrv } from '@grafana/runtime';
import { ElasticDatasource } from './datasource';
import { ElasticsearchOptions } from './types';
export function createElasticDatasource(
settings: Partial<DataSourceInstanceSettings<ElasticsearchOptions>> = {},
templateSrv: TemplateSrv
) {
export function createElasticDatasource(settings: Partial<DataSourceInstanceSettings<ElasticsearchOptions>> = {}) {
const { jsonData, ...rest } = settings;
const instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions> = {
@@ -48,5 +45,18 @@ export function createElasticDatasource(
...rest,
};
const templateSrv: TemplateSrv = {
getVariables: () => [],
replace: (text?: string) => {
if (text?.startsWith('$')) {
return `resolvedVariable`;
} else {
return text || '';
}
},
containsTemplate: (text?: string) => text?.includes('$') ?? false,
updateTimeRange: () => {},
};
return new ElasticDatasource(instanceSettings, templateSrv);
}