From 4ed0cc7d188d0a152ada1c21797cf34fbc1f8166 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Tue, 3 Jan 2023 12:38:11 -0600 Subject: [PATCH] Prometheus: Code editor - upgrade /series API endpoints to use label/values and /labels for supported prometheus clients (#59576) * Allow prometheus code editor API to use new prometheus API calls for supported data source clients. Co-authored-by: Ryan McKinley --- .../monaco-query-field/MonacoQueryField.tsx | 14 +++- .../monaco-completion-provider/completions.ts | 11 ++- .../prometheus/language_provider.test.ts | 72 +++++++++++++++++++ .../prometheus/language_provider.ts | 65 ++++++++++++++++- .../components/PromQueryBuilder.tsx | 2 +- 5 files changed, 150 insertions(+), 14 deletions(-) diff --git a/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx b/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx index 29adc1f3504..7952a7c0e82 100644 --- a/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx +++ b/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx @@ -141,8 +141,6 @@ const MonacoQueryField = (props: Props) => { }); // we construct a DataProvider object - const getSeries = (selector: string) => lpRef.current.getSeries(selector); - const getHistory = () => Promise.resolve(historyRef.current.map((h) => h.query.expr).filter((expr) => expr !== undefined)); @@ -164,7 +162,17 @@ const MonacoQueryField = (props: Props) => { const getLabelValues = (labelName: string) => lpRef.current.getLabelValues(labelName); - const dataProvider = { getSeries, getHistory, getAllMetricNames, getAllLabelNames, getLabelValues }; + const getSeriesValues = lpRef.current.getSeriesValues; + + const getSeriesLabels = lpRef.current.getSeriesLabels; + const dataProvider = { + getHistory, + getAllMetricNames, + getAllLabelNames, + getLabelValues, + getSeriesValues, + getSeriesLabels, + }; const completionProvider = getCompletionProvider(monaco, dataProvider); // completion-providers in monaco are not registered directly to editor-instances, diff --git a/public/app/plugins/datasource/prometheus/components/monaco-query-field/monaco-completion-provider/completions.ts b/public/app/plugins/datasource/prometheus/components/monaco-query-field/monaco-completion-provider/completions.ts index 08524be1340..7297ea1e6c6 100644 --- a/public/app/plugins/datasource/prometheus/components/monaco-query-field/monaco-completion-provider/completions.ts +++ b/public/app/plugins/datasource/prometheus/components/monaco-query-field/monaco-completion-provider/completions.ts @@ -27,7 +27,8 @@ export type DataProvider = { getAllMetricNames: () => Promise; getAllLabelNames: () => Promise; getLabelValues: (labelName: string) => Promise; - getSeries: (selector: string) => Promise>; + getSeriesValues: (name: string, match: string) => Promise; + getSeriesLabels: (selector: string, otherLabels: Label[]) => Promise; }; // we order items like: history, functions, metrics @@ -109,10 +110,7 @@ async function getLabelNames( return dataProvider.getAllLabelNames(); } else { const selector = makeSelector(metric, otherLabels); - const data = await dataProvider.getSeries(selector); - const possibleLabelNames = Object.keys(data); // all names from prometheus - const usedLabelNames = new Set(otherLabels.map((l) => l.name)); // names used in the query - return possibleLabelNames.filter((l) => !usedLabelNames.has(l)); + return await dataProvider.getSeriesLabels(selector, otherLabels); } } @@ -158,8 +156,7 @@ async function getLabelValues( return dataProvider.getLabelValues(labelName); } else { const selector = makeSelector(metric, otherLabels); - const data = await dataProvider.getSeries(selector); - return data[labelName] ?? []; + return await dataProvider.getSeriesValues(labelName, selector); } } diff --git a/public/app/plugins/datasource/prometheus/language_provider.test.ts b/public/app/plugins/datasource/prometheus/language_provider.test.ts index bdd2d4e4855..7bd00953db9 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.test.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.test.ts @@ -4,6 +4,7 @@ import Plain from 'slate-plain-serializer'; import { AbstractLabelOperator, HistoryItem } from '@grafana/data'; import { SearchFunctionType } from '@grafana/ui'; +import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; import { PrometheusDatasource } from './datasource'; import LanguageProvider from './language_provider'; import { PromQuery } from './types'; @@ -13,6 +14,7 @@ describe('Language completion provider', () => { metadataRequest: () => ({ data: { data: [] as any[] } }), getTimeRangeParams: () => ({ start: '0', end: '1' }), interpolateString: (string: string) => string, + hasLabelsMatchAPISupport: () => false, } as unknown as PrometheusDatasource; describe('cleanText', () => { @@ -66,6 +68,76 @@ describe('Language completion provider', () => { }); }); + describe('getSeriesLabels', () => { + it('should call series endpoint', () => { + const languageProvider = new LanguageProvider({ ...datasource } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { + end: '1', + 'match[]': '{job="grafana"}', + start: '0', + }); + }); + + it('should call labels endpoint', () => { + const languageProvider = new LanguageProvider({ + ...datasource, + hasLabelsMatchAPISupport: () => true, + } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith(`/api/v1/labels`, [], { + end: '1', + 'match[]': '{job="grafana"}', + start: '0', + }); + }); + }); + + describe('getSeriesValues', () => { + it('should call old series endpoint and should use match[] parameter', () => { + const languageProvider = new LanguageProvider(datasource); + const getSeriesValues = languageProvider.getSeriesValues; + const requestSpy = jest.spyOn(languageProvider, 'request'); + getSeriesValues('job', '{job="grafana"}'); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { + end: '1', + 'match[]': '{job="grafana"}', + start: '0', + }); + }); + + it('should call new series endpoint and should use match[] parameter', () => { + const languageProvider = new LanguageProvider({ + ...datasource, + hasLabelsMatchAPISupport: () => true, + } as PrometheusDatasource); + const getSeriesValues = languageProvider.getSeriesValues; + const requestSpy = jest.spyOn(languageProvider, 'request'); + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesValues(labelName, `{${labelName}="${labelValue}"}`); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith(`/api/v1/label/${labelName}/values`, [], { + end: '1', + 'match[]': `{${labelName}="${labelValue}"}`, + start: '0', + }); + }); + }); + describe('fetchSeries', () => { it('should use match[] parameter', () => { const languageProvider = new LanguageProvider(datasource); diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index d7a286fc6db..31970bb0db3 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -14,6 +14,7 @@ import { import { BackendSrvRequest } from '@grafana/runtime'; import { CompletionItem, CompletionItemGroup, SearchFunctionType, TypeaheadInput, TypeaheadOutput } from '@grafana/ui'; +import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; import { PrometheusDatasource } from './datasource'; import { addLimitInfo, @@ -98,6 +99,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { * 10 as a max size is totally arbitrary right now. */ private labelsCache = new LRU>({ max: 10 }); + private labelValuesCache = new LRU({ max: 10 }); constructor(datasource: PrometheusDatasource, initialValues?: Partial) { super(); @@ -501,19 +503,76 @@ export default class PromQlLanguageProvider extends LanguageProvider { return []; } + /** + * Gets series values + * Function to replace old getSeries calls in a way that will provide faster endpoints for new prometheus instances, + * while maintaining backward compatability + * @param labelName + * @param selector + */ + getSeriesValues = async (labelName: string, selector: string): Promise => { + if (!this.datasource.hasLabelsMatchAPISupport()) { + const data = await this.getSeries(selector); + return data[labelName] ?? []; + } + return await this.fetchSeriesValuesWithMatch(labelName, selector); + }; + /** * Fetches all values for a label, with optional match[] * @param name * @param match */ - fetchSeriesValues = async (name: string, match?: string): Promise => { + fetchSeriesValuesWithMatch = async (name: string, match?: string): Promise => { const interpolatedName = name ? this.datasource.interpolateString(name) : null; const range = this.datasource.getTimeRangeParams(); const urlParams = { ...range, - ...(interpolatedName && { 'match[]': match }), + ...(match && { 'match[]': match }), }; - return await this.request(`/api/v1/label/${interpolatedName}/values`, [], urlParams); + + const cacheParams = new URLSearchParams({ + 'match[]': interpolatedName ?? '', + start: roundSecToMin(parseInt(range.start, 10)).toString(), + end: roundSecToMin(parseInt(range.end, 10)).toString(), + name: name, + }); + + const cacheKey = `/api/v1/label/?${cacheParams.toString()}/values`; + let value: string[] | undefined = this.labelValuesCache.get(cacheKey); + if (!value) { + value = await this.request(`/api/v1/label/${interpolatedName}/values`, [], urlParams); + if (value) { + this.labelValuesCache.set(cacheKey, value); + } + } + return value ?? []; + }; + + /** + * Gets series labels + * Function to replace old getSeries calls in a way that will provide faster endpoints for new prometheus instances, + * while maintaining backward compatability. The old API call got the labels and the values in a single query, + * but with the new query we need two calls, one to get the labels, and another to get the values. + * + * @param selector + * @param otherLabels + */ + getSeriesLabels = async (selector: string, otherLabels: Label[]): Promise => { + let possibleLabelNames, data: Record; + + if (!this.datasource.hasLabelsMatchAPISupport()) { + data = await this.getSeries(selector); + possibleLabelNames = Object.keys(data); // all names from prometheus + } else { + // Exclude __name__ from output + otherLabels.push({ name: '__name__', value: '', op: '!=' }); + data = await this.fetchSeriesLabelsMatch(selector); + possibleLabelNames = Object.keys(data); + } + + const usedLabelNames = new Set(otherLabels.map((l) => l.name)); // names used in the query + return possibleLabelNames.filter((l) => !usedLabelNames.has(l)); }; /** diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx index 21b3b7469aa..78e54fe840d 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx @@ -157,7 +157,7 @@ export const PromQueryBuilder = React.memo((props) => { if (!forLabel.label) { return Promise.resolve([]); } - return datasource.languageProvider.fetchSeriesValues(forLabel.label, promQLExpression).then((response) => { + return datasource.languageProvider.fetchSeriesValuesWithMatch(forLabel.label, promQLExpression).then((response) => { return response.map((v) => ({ value: v, label: v,