From 7fa0ae48c5dcf415e3dee80375cd49dec9b91800 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:54:28 -0500 Subject: [PATCH] Loki: add detected_field/.../values support to language provider (#95204) * add detected_field values to language provider --- .../datasource/loki/LanguageProvider.test.ts | 107 +++++++++++++++++- .../datasource/loki/LanguageProvider.ts | 68 ++++++++++- .../createDetectedFieldsMetadataRequest.ts | 12 ++ .../app/plugins/datasource/loki/datasource.ts | 7 +- 4 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts diff --git a/public/app/plugins/datasource/loki/LanguageProvider.test.ts b/public/app/plugins/datasource/loki/LanguageProvider.test.ts index 1f07fd8a695..7ff7db74f7b 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.test.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.test.ts @@ -1,7 +1,8 @@ -import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRange } from '@grafana/data'; +import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRange, ScopedVars } from '@grafana/data'; import { config } from '@grafana/runtime'; import LanguageProvider from './LanguageProvider'; +import { createDetectedFieldsMetadataRequest } from './__mocks__/createDetectedFieldsMetadataRequest'; import { createLokiDatasource } from './__mocks__/datasource'; import { createMetadataRequest } from './__mocks__/metadataRequest'; import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource'; @@ -291,6 +292,94 @@ describe('Language completion provider', () => { }); }); + describe('fetchDetectedLabelValues', () => { + const expectedOptions = { + start: 1546372800000, + end: 1546380000000, + limit: 999, + }; + + const options: { + expr?: string; + timeRange?: TimeRange; + limit?: number; + scopedVars?: ScopedVars; + throwError?: boolean; + } = { + expr: '', + timeRange: mockTimeRange, + limit: 999, + throwError: true, + }; + + const labelName = 'labelName'; + + const datasource = detectedLabelValuesSetup(['label1_val1', 'label1_val2'], { + end: mockTimeRange.to.valueOf(), + start: mockTimeRange.from.valueOf(), + }); + + const expectedResponse = ['label1_val1', 'label1_val2']; + + it('should fetch detected label values if not cached', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + const labelValues = await provider.fetchDetectedLabelValues(labelName, options); + + expect(requestSpy).toHaveBeenCalledWith(`detected_field/${labelName}/values`, expectedOptions, true); + expect(labelValues).toEqual(expectedResponse); + }); + + it('should return cached values', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + const labelValues = await provider.fetchDetectedLabelValues(labelName, options); + + expect(requestSpy).toHaveBeenCalledTimes(1); + expect(labelValues).toEqual(expectedResponse); + + const nextLabelValues = await provider.fetchDetectedLabelValues(labelName, options); + expect(requestSpy).toHaveBeenCalledTimes(1); + expect(requestSpy).toHaveBeenCalledWith(`detected_field/${labelName}/values`, expectedOptions, true); + expect(nextLabelValues).toEqual(expectedResponse); + }); + + it('should encode special characters', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + await provider.fetchDetectedLabelValues('`\\"testkey', options); + + expect(requestSpy).toHaveBeenCalledWith('detected_field/%60%5C%22testkey/values', expectedOptions, true); + }); + + it('should cache by label name', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + + const promise1 = provider.fetchDetectedLabelValues('testkey', options); + const promise2 = provider.fetchDetectedLabelValues('testkey', options); + const datasource2 = detectedLabelValuesSetup(['label2_val1', 'label2_val2'], { + end: mockTimeRange.to.valueOf(), + start: mockTimeRange.from.valueOf(), + }); + const provider2 = await getLanguageProvider(datasource2, false); + const requestSpy2 = jest.spyOn(provider2, 'request'); + const promise3 = provider2.fetchDetectedLabelValues('testkeyNew', { ...options, expr: 'new' }); + + expect(requestSpy).toHaveBeenCalledTimes(1); + expect(requestSpy2).toHaveBeenCalledTimes(1); + + const values1 = await promise1; + const values2 = await promise2; + const values3 = await promise3; + + expect(values1).toStrictEqual(values2); + expect(values2).not.toStrictEqual(values3); + expect(requestSpy).toHaveBeenCalledTimes(1); + expect(requestSpy2).toHaveBeenCalledTimes(1); + }); + }); + describe('fetchLabels', () => { it('should return labels', async () => { const datasourceWithLabels = setup({ other: [] }); @@ -588,9 +677,11 @@ describe('Query imports', () => { }); }); -async function getLanguageProvider(datasource: LokiDatasource) { +async function getLanguageProvider(datasource: LokiDatasource, start = true) { const instance = new LanguageProvider(datasource); - await instance.start(); + if (start) { + await instance.start(); + } return instance; } @@ -611,3 +702,13 @@ function setup( return datasource; } + +function detectedLabelValuesSetup(response: string[], rangeMock: { start: number; end: number }): LokiDatasource { + const datasource = createLokiDatasource(); + + jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock); + jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldsMetadataRequest(response)); + jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string); + + return datasource; +} diff --git a/public/app/plugins/datasource/loki/LanguageProvider.ts b/public/app/plugins/datasource/loki/LanguageProvider.ts index ec72d57bafb..6b4dd6b6fcb 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.ts @@ -1,7 +1,7 @@ import { flatten } from 'lodash'; import { LRUCache } from 'lru-cache'; -import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange } from '@grafana/data'; +import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange, ScopedVars } from '@grafana/data'; import { config } from '@grafana/runtime'; import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource'; @@ -31,7 +31,9 @@ export default class LokiLanguageProvider extends LanguageProvider { */ private seriesCache = new LRUCache>({ max: 10 }); private labelsCache = new LRUCache({ max: 10 }); + private detectedFieldValuesCache = new LRUCache({ max: 10 }); private labelsPromisesCache = new LRUCache>({ max: 10 }); + private detectedLabelValuesPromisesCache = new LRUCache>({ max: 10 }); constructor(datasource: LokiDatasource, initialValues?: any) { super(); @@ -42,11 +44,14 @@ export default class LokiLanguageProvider extends LanguageProvider { Object.assign(this, initialValues); } - request = async (url: string, params?: Record) => { + request = async (url: string, params?: Record, throwError?: boolean) => { try { return await this.datasource.metadataRequest(url, params); } catch (error) { console.error(error); + if (throwError) { + throw error; + } } return undefined; @@ -238,6 +243,65 @@ export default class LokiLanguageProvider extends LanguageProvider { return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0; } + async fetchDetectedLabelValues( + labelName: string, + options?: { expr?: string; timeRange?: TimeRange; limit?: number; scopedVars?: ScopedVars; throwError?: boolean } + ): Promise { + const label = encodeURIComponent(this.datasource.interpolateString(labelName)); + + const interpolatedExpr = + options?.expr && options.expr !== EMPTY_SELECTOR + ? this.datasource.interpolateString(options.expr, options.scopedVars) + : undefined; + + const url = `detected_field/${label}/values`; + const range = options?.timeRange ?? this.getDefaultTimeRange(); + const rangeParams = this.datasource.getTimeRangeParams(range); + const { start, end } = rangeParams; + const params: KeyValue = { start, end, limit: options?.limit ?? 1000 }; + let paramCacheKey = label; + + if (interpolatedExpr) { + params.query = interpolatedExpr; + paramCacheKey += interpolatedExpr; + } + + const cacheKey = this.generateCacheKey(url, start, end, paramCacheKey); + + // Values in cache, return + const labelValues = this.detectedFieldValuesCache.get(cacheKey); + if (labelValues) { + return labelValues; + } + + // Promise in cache, return + let labelValuesPromise = this.detectedLabelValuesPromisesCache.get(cacheKey); + if (labelValuesPromise) { + return labelValuesPromise; + } + + labelValuesPromise = new Promise(async (resolve) => { + try { + const data = await this.request(url, params, options?.throwError); + if (Array.isArray(data)) { + const labelValues = data.slice().sort(); + this.detectedFieldValuesCache.set(cacheKey, labelValues); + this.detectedLabelValuesPromisesCache.delete(cacheKey); + resolve(labelValues); + } + } catch (error) { + console.error(error); + resolve([]); + if (options?.throwError) { + throw error; + } + } + }); + this.detectedLabelValuesPromisesCache.set(cacheKey, labelValuesPromise); + + return labelValuesPromise; + } + /** * Fetch label values * diff --git a/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts new file mode 100644 index 00000000000..73d0388dfb2 --- /dev/null +++ b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts @@ -0,0 +1,12 @@ +export function createDetectedFieldsMetadataRequest(labelsAndValues: string[]) { + const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/; + + return async function metadataRequestMock(url: string) { + const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex); + if (labelsMatch) { + return labelsAndValues ?? []; + } else { + throw new Error(`Unexpected url error, ${url}`); + } + }; +} diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index d7f5862159f..f661d8d5f39 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -541,7 +541,12 @@ export class LokiDatasource } const res = await this.getResource(url, params, options); - return res.data || []; + + // detected_field/${label}/values has different structure then other metadata responses + if (!res.data && res.values) { + return res.values ?? []; + } + return res.data ?? []; } /**