From 1758ddd457f1aadf5d4db834a6ea9c96710ffec0 Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Mon, 12 Dec 2022 04:01:27 -0800 Subject: [PATCH] Scenes: Add query variable support (#59553) * WIP first attempt to query variable * regex issue repro demo * Refresh variable on time range change if refresh specified * Instantiate variable runner when updating query variable options * Simplify runners getTarget interface * Fix issue with variable ot being updated correctly after other variable changed * Add templateSrv.replace compatibility with query variable * QueryVariable: use datasource variable as source * use proper format * Make sure variables set is correctly updated when query variable errors * Do not destruct scopedVars when using sceneGraph.interpolate in templateSrv * Add support for Legacy variables (metricFindQuery) * Review * Fix lint * Test: Add unit for datasource by variable * test: Add unit for datasource as var * query: delegate interpolation to datasourceSrv * Cleanup Co-authored-by: Ivan Ortega --- .betterer.results | 21 +- public/app/features/scenes/scenes/index.tsx | 2 + .../scenes/scenes/queryVariableDemo.tsx | 71 ++++ .../features/scenes/scenes/variablesDemo.tsx | 2 +- .../interpolation/ScopedVarsVariable.ts | 4 +- .../variables/interpolation/formatRegistry.ts | 2 + .../interpolation/sceneInterpolator.test.ts | 1 + .../interpolation/sceneInterpolator.ts | 11 +- .../scenes/variables/sets/SceneVariableSet.ts | 23 +- public/app/features/scenes/variables/types.ts | 2 + .../variables/variants/ConstantVariable.ts | 8 + .../variables/variants/CustomVariable.tsx | 2 +- .../variables/variants/DataSourceVariable.tsx | 1 + .../variants/MultiValueVariable.test.ts | 11 + .../variables/variants/ObjectVariable.test.ts | 1 + .../variables/variants/TestVariable.tsx | 1 + .../variants/query/QueryVariable.test.tsx | 310 ++++++++++++++++++ .../variants/query/QueryVariable.tsx | 166 ++++++++++ .../query/createQueryVariableRunner.ts | 115 +++++++ .../scenes/variables/variants/query/utils.ts | 111 +++++++ .../templating/LegacyVariableWrapper.ts | 17 +- .../features/templating/template_srv.test.ts | 24 ++ .../app/features/templating/template_srv.ts | 14 +- public/test/setupTests.ts | 2 +- 24 files changed, 894 insertions(+), 28 deletions(-) create mode 100644 public/app/features/scenes/scenes/queryVariableDemo.tsx create mode 100644 public/app/features/scenes/variables/variants/query/QueryVariable.test.tsx create mode 100644 public/app/features/scenes/variables/variants/query/QueryVariable.tsx create mode 100644 public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts create mode 100644 public/app/features/scenes/variables/variants/query/utils.ts diff --git a/.betterer.results b/.betterer.results index 32bd5512dd8..0799803bd64 100644 --- a/.betterer.results +++ b/.betterer.results @@ -4575,6 +4575,9 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "3"], [0, 0, 0, "Do not use any type assertions.", "4"] ], + "public/app/features/scenes/variables/interpolation/sceneInterpolator.ts:5381": [ + [0, 0, 0, "Do not use any type assertions.", "0"] + ], "public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], @@ -4588,6 +4591,16 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"] ], + "public/app/features/scenes/variables/variants/query/QueryVariable.tsx:5381": [ + [0, 0, 0, "Unexpected any. Specify a different type.", "0"] + ], + "public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts:5381": [ + [0, 0, 0, "Unexpected any. Specify a different type.", "0"] + ], + "public/app/features/scenes/variables/variants/query/utils.ts:5381": [ + [0, 0, 0, "Unexpected any. Specify a different type.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "1"] + ], "public/app/features/search/components/SearchCard.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"], @@ -4692,7 +4705,8 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "1"], [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"], - [0, 0, 0, "Unexpected any. Specify a different type.", "4"] + [0, 0, 0, "Unexpected any. Specify a different type.", "4"], + [0, 0, 0, "Unexpected any. Specify a different type.", "5"] ], "public/app/features/templating/template_srv.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], @@ -4712,8 +4726,9 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "14"], [0, 0, 0, "Unexpected any. Specify a different type.", "15"], [0, 0, 0, "Unexpected any. Specify a different type.", "16"], - [0, 0, 0, "Unexpected any. Specify a different type.", "17"], - [0, 0, 0, "Do not use any type assertions.", "18"] + [0, 0, 0, "Do not use any type assertions.", "17"], + [0, 0, 0, "Unexpected any. Specify a different type.", "18"], + [0, 0, 0, "Do not use any type assertions.", "19"] ], "public/app/features/transformers/FilterByValueTransformer/ValueMatchers/BasicMatcherEditor.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/public/app/features/scenes/scenes/index.tsx b/public/app/features/scenes/scenes/index.tsx index 94b2c479334..65189c47175 100644 --- a/public/app/features/scenes/scenes/index.tsx +++ b/public/app/features/scenes/scenes/index.tsx @@ -7,6 +7,7 @@ import { getMultipleGridLayoutTest } from './gridMultiple'; import { getGridWithMultipleData } from './gridWithMultipleData'; import { getGridWithRowLayoutTest } from './gridWithRow'; import { getNestedScene } from './nested'; +import { getQueryVariableDemo } from './queryVariableDemo'; import { getSceneWithRows } from './sceneWithRows'; import { getVariablesDemo } from './variablesDemo'; @@ -26,6 +27,7 @@ export function getScenes(): SceneDef[] { { title: 'Grid with rows and different queries and time ranges', getScene: getGridWithMultipleTimeRanges }, { title: 'Multiple grid layouts test', getScene: getMultipleGridLayoutTest }, { title: 'Variables', getScene: getVariablesDemo }, + { title: 'Query variable', getScene: getQueryVariableDemo }, ]; } diff --git a/public/app/features/scenes/scenes/queryVariableDemo.tsx b/public/app/features/scenes/scenes/queryVariableDemo.tsx new file mode 100644 index 00000000000..ad209a5c449 --- /dev/null +++ b/public/app/features/scenes/scenes/queryVariableDemo.tsx @@ -0,0 +1,71 @@ +import { VariableRefresh } from '@grafana/data'; + +import { Scene, EmbeddedScene } from '../components/Scene'; +import { SceneCanvasText } from '../components/SceneCanvasText'; +import { SceneSubMenu } from '../components/SceneSubMenu'; +import { SceneTimePicker } from '../components/SceneTimePicker'; +import { SceneFlexLayout } from '../components/layout/SceneFlexLayout'; +import { SceneTimeRange } from '../core/SceneTimeRange'; +import { VariableValueSelectors } from '../variables/components/VariableValueSelectors'; +import { SceneVariableSet } from '../variables/sets/SceneVariableSet'; +import { CustomVariable } from '../variables/variants/CustomVariable'; +import { DataSourceVariable } from '../variables/variants/DataSourceVariable'; +import { QueryVariable } from '../variables/variants/query/QueryVariable'; + +export function getQueryVariableDemo(standalone: boolean): Scene { + const state = { + title: 'Query variable', + $variables: new SceneVariableSet({ + variables: [ + new CustomVariable({ + name: 'metric', + query: 'job : job, instance : instance', + }), + new DataSourceVariable({ + name: 'datasource', + query: 'prometheus', + }), + new QueryVariable({ + name: 'instance (using datasource variable)', + refresh: VariableRefresh.onTimeRangeChanged, + query: { query: 'label_values(go_gc_duration_seconds, ${metric})' }, + datasource: '${datasource}', + }), + new QueryVariable({ + name: 'label values (on time range refresh)', + refresh: VariableRefresh.onTimeRangeChanged, + query: { query: 'label_values(go_gc_duration_seconds, ${metric})' }, + datasource: { uid: 'gdev-prometheus', type: 'prometheus' }, + }), + new QueryVariable({ + name: 'legacy (graphite)', + refresh: VariableRefresh.onTimeRangeChanged, + query: { queryType: 'Default', target: 'stats.response.*' }, + datasource: { uid: 'gdev-graphite', type: 'graphite' }, + }), + ], + }), + layout: new SceneFlexLayout({ + direction: 'row', + children: [ + new SceneFlexLayout({ + children: [ + new SceneCanvasText({ + size: { width: '40%' }, + text: 'metric: ${metric}', + fontSize: 20, + align: 'center', + }), + ], + }), + ], + }), + $timeRange: new SceneTimeRange(), + actions: [new SceneTimePicker({})], + subMenu: new SceneSubMenu({ + children: [new VariableValueSelectors({})], + }), + }; + + return standalone ? new Scene(state) : new EmbeddedScene(state); +} diff --git a/public/app/features/scenes/scenes/variablesDemo.tsx b/public/app/features/scenes/scenes/variablesDemo.tsx index b5280456127..5849de39481 100644 --- a/public/app/features/scenes/scenes/variablesDemo.tsx +++ b/public/app/features/scenes/scenes/variablesDemo.tsx @@ -1,5 +1,5 @@ import { VizPanel } from '../components'; -import { EmbeddedScene, Scene } from '../components/Scene'; +import { Scene, EmbeddedScene } from '../components/Scene'; import { SceneCanvasText } from '../components/SceneCanvasText'; import { SceneSubMenu } from '../components/SceneSubMenu'; import { SceneTimePicker } from '../components/SceneTimePicker'; diff --git a/public/app/features/scenes/variables/interpolation/ScopedVarsVariable.ts b/public/app/features/scenes/variables/interpolation/ScopedVarsVariable.ts index 1b724265d7a..d3343d87be4 100644 --- a/public/app/features/scenes/variables/interpolation/ScopedVarsVariable.ts +++ b/public/app/features/scenes/variables/interpolation/ScopedVarsVariable.ts @@ -9,10 +9,10 @@ import { FormatVariable } from './formatRegistry'; export class ScopedVarsVariable implements FormatVariable { private static fieldAccessorCache: FieldAccessorCache = {}; - public state: { name: string; value: ScopedVar }; + public state: { name: string; value: ScopedVar; type: string }; public constructor(name: string, value: ScopedVar) { - this.state = { name, value }; + this.state = { name, value, type: 'scopedvar' }; } public getValue(fieldPath: string): VariableValue { diff --git a/public/app/features/scenes/variables/interpolation/formatRegistry.ts b/public/app/features/scenes/variables/interpolation/formatRegistry.ts index 3d4e3fe6148..6571d5decaf 100644 --- a/public/app/features/scenes/variables/interpolation/formatRegistry.ts +++ b/public/app/features/scenes/variables/interpolation/formatRegistry.ts @@ -1,6 +1,7 @@ import { isArray, map, replace } from 'lodash'; import { dateTime, Registry, RegistryItem, textUtil } from '@grafana/data'; +import { VariableType } from '@grafana/schema'; import kbn from 'app/core/utils/kbn'; import { ALL_VARIABLE_VALUE } from 'app/features/variables/constants'; @@ -18,6 +19,7 @@ export interface FormatRegistryItem extends RegistryItem { export interface FormatVariable { state: { name: string; + type: VariableType | string; }; getValue(fieldPath?: string): VariableValue | undefined | null; diff --git a/public/app/features/scenes/variables/interpolation/sceneInterpolator.test.ts b/public/app/features/scenes/variables/interpolation/sceneInterpolator.test.ts index ac99a9e38c5..4ba14ce5503 100644 --- a/public/app/features/scenes/variables/interpolation/sceneInterpolator.test.ts +++ b/public/app/features/scenes/variables/interpolation/sceneInterpolator.test.ts @@ -51,6 +51,7 @@ describe('sceneInterpolator', () => { $variables: new SceneVariableSet({ variables: [ new ObjectVariable({ + type: 'custom', name: 'test', value: { prop1: 'prop1Value' }, }), diff --git a/public/app/features/scenes/variables/interpolation/sceneInterpolator.ts b/public/app/features/scenes/variables/interpolation/sceneInterpolator.ts index 8b72af1ef95..c9fb8d4bee7 100644 --- a/public/app/features/scenes/variables/interpolation/sceneInterpolator.ts +++ b/public/app/features/scenes/variables/interpolation/sceneInterpolator.ts @@ -1,5 +1,5 @@ import { ScopedVars } from '@grafana/data'; -import { VariableModel } from '@grafana/schema'; +import { VariableModel, VariableType } from '@grafana/schema'; import { variableRegex } from 'app/features/variables/utils'; import { EmptyVariableSet, sceneGraph } from '../../core/sceneGraph'; @@ -12,7 +12,7 @@ import { formatRegistry, FormatRegistryID, FormatVariable } from './formatRegist export type CustomFormatterFn = ( value: unknown, legacyVariableModel: VariableModel, - legacyDefaultFormatter: CustomFormatterFn + legacyDefaultFormatter?: CustomFormatterFn ) => string; /** @@ -97,9 +97,10 @@ function formatValue( } if (typeof formatNameOrFn === 'function') { - // legacy custom formatter function, TODO - //return format(value, {}, this.formatValue); - throw new Error('Custom formatter function not supported'); + return formatNameOrFn(value, { + name: variable.state.name, + type: variable.state.type as VariableType, + }); } let args: string[] = []; diff --git a/public/app/features/scenes/variables/sets/SceneVariableSet.ts b/public/app/features/scenes/variables/sets/SceneVariableSet.ts index 32388dc83c9..76316da53bb 100644 --- a/public/app/features/scenes/variables/sets/SceneVariableSet.ts +++ b/public/app/features/scenes/variables/sets/SceneVariableSet.ts @@ -40,7 +40,7 @@ export class SceneVariableSet extends SceneObjectBase imp this.variablesToUpdate.clear(); for (const update of this.updating.values()) { - update.subscription.unsubscribe(); + update.subscription?.unsubscribe(); } } @@ -70,12 +70,14 @@ export class SceneVariableSet extends SceneObjectBase imp continue; } - this.updating.set(variable, { + const variableToUpdate: VariableUpdateInProgress = { variable, - subscription: variable.validateAndUpdate().subscribe({ - next: () => this.validateAndUpdateCompleted(variable), - error: (err) => this.handleVariableError(variable, err), - }), + }; + + this.updating.set(variable, variableToUpdate); + variableToUpdate.subscription = variable.validateAndUpdate().subscribe({ + next: () => this.validateAndUpdateCompleted(variable), + error: (err) => this.handleVariableError(variable, err), }); } } @@ -85,7 +87,7 @@ export class SceneVariableSet extends SceneObjectBase imp */ private validateAndUpdateCompleted(variable: SceneVariable) { const update = this.updating.get(variable); - update?.subscription.unsubscribe(); + update?.subscription?.unsubscribe(); this.updating.delete(variable); this.variablesToUpdate.delete(variable); @@ -97,6 +99,11 @@ export class SceneVariableSet extends SceneObjectBase imp * Not sure if this should be handled here on in MultiValueVariable */ private handleVariableError(variable: SceneVariable, err: Error) { + const update = this.updating.get(variable); + update?.subscription?.unsubscribe(); + + this.updating.delete(variable); + this.variablesToUpdate.delete(variable); variable.setState({ loading: false, error: err }); } @@ -187,5 +194,5 @@ export class SceneVariableSet extends SceneObjectBase imp export interface VariableUpdateInProgress { variable: SceneVariable; - subscription: Unsubscribable; + subscription?: Unsubscribable; } diff --git a/public/app/features/scenes/variables/types.ts b/public/app/features/scenes/variables/types.ts index d845bffcb85..a7ba99a6da9 100644 --- a/public/app/features/scenes/variables/types.ts +++ b/public/app/features/scenes/variables/types.ts @@ -1,11 +1,13 @@ import { Observable } from 'rxjs'; import { BusEventWithPayload } from '@grafana/data'; +import { VariableType } from '@grafana/schema'; import { VariableHide } from 'app/features/variables/types'; import { SceneObject, SceneObjectStatePlain } from '../core/types'; export interface SceneVariableState extends SceneObjectStatePlain { + type: VariableType; name: string; label?: string; hide?: VariableHide; diff --git a/public/app/features/scenes/variables/variants/ConstantVariable.ts b/public/app/features/scenes/variables/variants/ConstantVariable.ts index 05f97a342bf..f946cc9b13e 100644 --- a/public/app/features/scenes/variables/variants/ConstantVariable.ts +++ b/public/app/features/scenes/variables/variants/ConstantVariable.ts @@ -9,6 +9,14 @@ export class ConstantVariable extends SceneObjectBase implements SceneVariable { + public constructor(initialState: Partial) { + super({ + type: 'constant', + value: '', + name: '', + ...initialState, + }); + } public getValue(): VariableValue { return this.state.value; } diff --git a/public/app/features/scenes/variables/variants/CustomVariable.tsx b/public/app/features/scenes/variables/variants/CustomVariable.tsx index 07c923b7d54..3237bd8efb9 100644 --- a/public/app/features/scenes/variables/variants/CustomVariable.tsx +++ b/public/app/features/scenes/variables/variants/CustomVariable.tsx @@ -19,6 +19,7 @@ export class CustomVariable extends MultiValueVariable { public constructor(initialState: Partial) { super({ + type: 'custom', query: '', value: '', text: '', @@ -30,7 +31,6 @@ export class CustomVariable extends MultiValueVariable { public getValueOptions(args: VariableGetOptionsArgs): Observable { const match = this.state.query.match(/(?:\\,|[^,])+/g) ?? []; - const options = match.map((text) => { text = text.replace(/\\,/g, ','); const textMatch = /^(.+)\s:\s(.+)$/g.exec(text) ?? []; diff --git a/public/app/features/scenes/variables/variants/DataSourceVariable.tsx b/public/app/features/scenes/variables/variants/DataSourceVariable.tsx index 18712274f0c..10ee283a5f4 100644 --- a/public/app/features/scenes/variables/variants/DataSourceVariable.tsx +++ b/public/app/features/scenes/variables/variants/DataSourceVariable.tsx @@ -24,6 +24,7 @@ export class DataSourceVariable extends MultiValueVariable) { super({ + type: 'datasource', value: '', text: '', options: [], diff --git a/public/app/features/scenes/variables/variants/MultiValueVariable.test.ts b/public/app/features/scenes/variables/variants/MultiValueVariable.test.ts index 8a9dfbb9205..0c1df86be53 100644 --- a/public/app/features/scenes/variables/variants/MultiValueVariable.test.ts +++ b/public/app/features/scenes/variables/variants/MultiValueVariable.test.ts @@ -10,6 +10,17 @@ export interface ExampleVariableState extends MultiValueVariableState { } class ExampleVariable extends MultiValueVariable { + public constructor(initialState: Partial) { + super({ + type: 'custom', + optionsToReturn: [], + value: '', + text: '', + name: '', + options: [], + ...initialState, + }); + } public getValueOptions(args: VariableGetOptionsArgs): Observable { return of(this.state.optionsToReturn); } diff --git a/public/app/features/scenes/variables/variants/ObjectVariable.test.ts b/public/app/features/scenes/variables/variants/ObjectVariable.test.ts index cec60fe49d5..3dfbcc08a40 100644 --- a/public/app/features/scenes/variables/variants/ObjectVariable.test.ts +++ b/public/app/features/scenes/variables/variants/ObjectVariable.test.ts @@ -5,6 +5,7 @@ describe('ObjectVariable', () => { it('it should return value according to fieldPath', () => { const variable = new ObjectVariable({ name: 'test', + type: 'custom', value: { field1: 'value1', array: ['value1', 'value2', 'value3'], diff --git a/public/app/features/scenes/variables/variants/TestVariable.tsx b/public/app/features/scenes/variables/variants/TestVariable.tsx index 10601be6e9e..c7fd3afe5e5 100644 --- a/public/app/features/scenes/variables/variants/TestVariable.tsx +++ b/public/app/features/scenes/variables/variants/TestVariable.tsx @@ -31,6 +31,7 @@ export class TestVariable extends MultiValueVariable { public constructor(initialState: Partial) { super({ + type: 'custom', name: 'Test', value: 'Value', text: 'Text', diff --git a/public/app/features/scenes/variables/variants/query/QueryVariable.test.tsx b/public/app/features/scenes/variables/variants/query/QueryVariable.test.tsx new file mode 100644 index 00000000000..d14375ef688 --- /dev/null +++ b/public/app/features/scenes/variables/variants/query/QueryVariable.test.tsx @@ -0,0 +1,310 @@ +import { lastValueFrom, of } from 'rxjs'; + +import { + DataQueryRequest, + DataSourceApi, + DataSourceRef, + FieldType, + getDefaultTimeRange, + LoadingState, + PanelData, + PluginType, + StandardVariableSupport, + toDataFrame, + toUtc, + VariableRefresh, + VariableSupportType, +} from '@grafana/data'; +import { SceneFlexLayout } from 'app/features/scenes/components'; +import { SceneTimeRange } from 'app/features/scenes/core/SceneTimeRange'; + +import { SceneVariableSet } from '../../sets/SceneVariableSet'; + +import { QueryVariable } from './QueryVariable'; +import { QueryRunner, RunnerArgs, setCreateQueryVariableRunnerFactory } from './createQueryVariableRunner'; + +const runRequestMock = jest.fn().mockReturnValue( + of({ + state: LoadingState.Done, + series: [ + toDataFrame({ + fields: [{ name: 'text', type: FieldType.string, values: ['A', 'AB', 'C'] }], + }), + ], + timeRange: getDefaultTimeRange(), + }) +); + +const fakeDsMock: DataSourceApi = { + name: 'fake-std', + type: 'fake-std', + getRef: () => ({ type: 'fake-std', uid: 'fake-std' }), + query: () => + Promise.resolve({ + data: [], + }), + testDatasource: () => Promise.resolve({ status: 'success' }), + meta: { + id: 'fake-std', + type: PluginType.datasource, + module: 'fake-std', + baseUrl: '', + name: 'fake-std', + info: { + author: { name: '' }, + description: '', + links: [], + logos: { large: '', small: '' }, + updated: '', + version: '', + screenshots: [], + }, + }, + // Standard variable support + variables: { + getType: () => VariableSupportType.Standard, + toDataQuery: (q) => ({ ...q, refId: 'FakeDataSource-refId' }), + }, + id: 1, + uid: 'fake-std', +}; + +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => ({ + get: (ds: DataSourceRef): Promise => { + return Promise.resolve(fakeDsMock); + }, + }), +})); + +class FakeQueryRunner implements QueryRunner { + public constructor(private datasource: DataSourceApi, private _runRequest: jest.Mock) {} + + public getTarget(variable: QueryVariable) { + return (this.datasource.variables as StandardVariableSupport).toDataQuery(variable.state.query); + } + public runRequest(args: RunnerArgs, request: DataQueryRequest) { + return this._runRequest( + this.datasource, + request, + (this.datasource.variables as StandardVariableSupport).query + ); + } +} + +describe('QueryVariable', () => { + describe('When empty query is provided', () => { + it('Should default to empty options and empty value', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake', type: 'fake' }, + query: '', + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(variable.state.value).toEqual(''); + expect(variable.state.text).toEqual(''); + expect(variable.state.options).toEqual([]); + }); + }); + + describe('When no data source is provided', () => { + it('Should default to empty options and empty value', async () => { + const variable = new QueryVariable({ + name: 'test', + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(variable.state.value).toEqual(''); + expect(variable.state.text).toEqual(''); + expect(variable.state.options).toEqual([]); + }); + }); + + describe('Issuing variable query', () => { + const originalNow = Date.now; + beforeEach(() => { + setCreateQueryVariableRunnerFactory(() => new FakeQueryRunner(fakeDsMock, runRequestMock)); + }); + + beforeEach(() => { + Date.now = jest.fn(() => 60000); + }); + + afterEach(() => { + Date.now = originalNow; + runRequestMock.mockClear(); + }); + + it('Should resolve variable options via provided runner', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(variable.state.options).toEqual([ + { label: 'A', value: 'A' }, + { label: 'AB', value: 'AB' }, + { label: 'C', value: 'C' }, + ]); + }); + + it('Should pass variable scene object via request scoped vars', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + }); + + await lastValueFrom(variable.validateAndUpdate()); + const call = runRequestMock.mock.calls[0]; + expect(call[1].scopedVars.__sceneObject).toEqual({ value: variable, text: '__sceneObject' }); + }); + + describe('when refresh on dashboard load set', () => { + it('Should issue variable query with default time range', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(runRequestMock).toBeCalledTimes(1); + const call = runRequestMock.mock.calls[0]; + expect(call[1].range).toEqual(getDefaultTimeRange()); + }); + + it('Should not issue variable query when the closest time range changes if refresh on dahboard load is set', async () => { + const timeRange = new SceneTimeRange({ from: 'now-1h', to: 'now' }); + + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + refresh: VariableRefresh.onDashboardLoad, + $timeRange: timeRange, + }); + + variable.activate(); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(runRequestMock).toBeCalledTimes(1); + const call1 = runRequestMock.mock.calls[0]; + + // Uses default time range + expect(call1[1].range.raw).toEqual({ + from: 'now-6h', + to: 'now', + }); + + timeRange.onTimeRangeChange({ + from: toUtc('2020-01-01'), + to: toUtc('2020-01-02'), + raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') }, + }); + + await Promise.resolve(); + + expect(runRequestMock).toBeCalledTimes(1); + }); + }); + + describe('when refresh on time range change set', () => { + it('Should issue variable query with closes time range if refresh on time range change set', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + refresh: VariableRefresh.onTimeRangeChanged, + }); + + // @ts-expect-error + const scene = new SceneFlexLayout({ + $timeRange: new SceneTimeRange({ from: 'now-1h', to: 'now' }), + $variables: new SceneVariableSet({ + variables: [variable], + }), + children: [], + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(runRequestMock).toBeCalledTimes(1); + const call = runRequestMock.mock.calls[0]; + + expect(call[1].range.raw).toEqual({ + from: 'now-1h', + to: 'now', + }); + }); + + it('Should issue variable query when time range changes if refresh on time range change is set', async () => { + const timeRange = new SceneTimeRange({ from: 'now-1h', to: 'now' }); + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + refresh: VariableRefresh.onTimeRangeChanged, + $timeRange: timeRange, + }); + + variable.activate(); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(runRequestMock).toBeCalledTimes(1); + const call1 = runRequestMock.mock.calls[0]; + expect(call1[1].range.raw).toEqual({ + from: 'now-1h', + to: 'now', + }); + + timeRange.onTimeRangeChange({ + from: toUtc('2020-01-01'), + to: toUtc('2020-01-02'), + raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') }, + }); + + await new Promise((r) => setTimeout(r, 1)); + + expect(runRequestMock).toBeCalledTimes(2); + const call2 = runRequestMock.mock.calls[1]; + expect(call2[1].range.raw).toEqual({ + from: '2020-01-01T00:00:00.000Z', + to: '2020-01-02T00:00:00.000Z', + }); + }); + }); + }); + + describe('When regex provided', () => { + beforeEach(() => { + setCreateQueryVariableRunnerFactory(() => new FakeQueryRunner(fakeDsMock, runRequestMock)); + }); + + it('should return options that match regex', async () => { + const variable = new QueryVariable({ + name: 'test', + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + regex: '/^A/', + }); + + await lastValueFrom(variable.validateAndUpdate()); + + expect(variable.state.options).toEqual([ + { label: 'A', value: 'A' }, + { label: 'AB', value: 'AB' }, + ]); + }); + }); +}); diff --git a/public/app/features/scenes/variables/variants/query/QueryVariable.tsx b/public/app/features/scenes/variables/variants/query/QueryVariable.tsx new file mode 100644 index 00000000000..b090851c9ba --- /dev/null +++ b/public/app/features/scenes/variables/variants/query/QueryVariable.tsx @@ -0,0 +1,166 @@ +import React from 'react'; +import { Observable, Subject, of, Unsubscribable, filter, take, mergeMap, catchError, throwError, from } from 'rxjs'; +import { v4 as uuidv4 } from 'uuid'; + +import { + CoreApp, + DataQuery, + DataQueryRequest, + DataSourceApi, + DataSourceRef, + getDefaultTimeRange, + LoadingState, + PanelData, + ScopedVars, + VariableRefresh, + VariableSort, +} from '@grafana/data'; +import { getDataSourceSrv } from '@grafana/runtime'; +import { toMetricFindValues } from 'app/features/variables/query/operators'; + +import { sceneGraph } from '../../../core/sceneGraph'; +import { SceneComponentProps } from '../../../core/types'; +import { VariableDependencyConfig } from '../../VariableDependencyConfig'; +import { VariableValueSelect } from '../../components/VariableValueSelect'; +import { VariableValueOption } from '../../types'; +import { MultiValueVariable, MultiValueVariableState, VariableGetOptionsArgs } from '../MultiValueVariable'; + +import { createQueryVariableRunner } from './createQueryVariableRunner'; +import { metricNamesToVariableValues } from './utils'; + +export interface QueryVariableState extends MultiValueVariableState { + type: 'query'; + datasource: DataSourceRef | string | null; + query: any; + regex: string; + refresh: VariableRefresh; + sort: VariableSort; +} + +export class QueryVariable extends MultiValueVariable { + private updateSubscription?: Unsubscribable; + private dataSourceSubject?: Subject; + + protected _variableDependency = new VariableDependencyConfig(this, { + statePaths: ['regex', 'query', 'datasource'], + }); + + public constructor(initialState: Partial) { + super({ + type: 'query', + name: '', + value: '', + text: '', + query: '', + options: [], + datasource: null, + regex: '', + refresh: VariableRefresh.onDashboardLoad, + sort: VariableSort.alphabeticalAsc, + ...initialState, + }); + } + + public activate(): void { + super.activate(); + const timeRange = sceneGraph.getTimeRange(this); + + if (this.state.refresh === VariableRefresh.onTimeRangeChanged) { + this._subs.add( + timeRange.subscribeToState({ + next: () => { + this.updateSubscription = this.validateAndUpdate().subscribe(); + }, + }) + ); + } + } + + public deactivate(): void { + super.deactivate(); + if (this.updateSubscription) { + this.updateSubscription.unsubscribe(); + } + + if (this.dataSourceSubject) { + this.dataSourceSubject.unsubscribe(); + } + } + + public getValueOptions(args: VariableGetOptionsArgs): Observable { + if (this.state.query === '' || !this.state.datasource) { + return of([]); + } + + return from(this.getDataSource()).pipe( + mergeMap((ds) => { + const runner = createQueryVariableRunner(ds); + const target = runner.getTarget(this); + const request = this.getRequest(target); + return runner.runRequest({ variable: this }, request).pipe( + filter((data) => data.state === LoadingState.Done || data.state === LoadingState.Error), // we only care about done or error for now + take(1), // take the first result, using first caused a bug where it in some situations throw an uncaught error because of no results had been received yet + mergeMap((data: PanelData) => { + if (data.state === LoadingState.Error) { + return throwError(() => data.error); + } + return of(data); + }), + toMetricFindValues(), + mergeMap((values) => { + let regex = ''; + if (this.state.regex) { + regex = sceneGraph.interpolate(this, this.state.regex, undefined, 'regex'); + } + return of(metricNamesToVariableValues(regex, this.state.sort, values)); + }), + catchError((error) => { + if (error.cancelled) { + return of([]); + } + return throwError(() => error); + }) + ); + }) + ); + } + + private async getDataSource(): Promise { + return getDataSourceSrv().get(this.state.datasource ?? '', { + __sceneObject: { text: '__sceneObject', value: this }, + }); + } + + private getRequest(target: DataQuery) { + // TODO: add support for search filter + // const { searchFilter } = this.state.searchFilter; + // const searchFilterScope = { searchFilter: { text: searchFilter, value: searchFilter } }; + // const searchFilterAsVars = searchFilter ? searchFilterScope : {}; + const scopedVars: ScopedVars = { + // ...searchFilterAsVars, + __sceneObject: { text: '__sceneObject', value: this }, + }; + + const range = + this.state.refresh === VariableRefresh.onTimeRangeChanged + ? sceneGraph.getTimeRange(this).state.value + : getDefaultTimeRange(); + + const request: DataQueryRequest = { + app: CoreApp.Dashboard, + requestId: uuidv4(), + timezone: '', + range, + interval: '', + intervalMs: 0, + targets: [target], + scopedVars, + startTime: Date.now(), + }; + return request; + } + + public static Component = ({ model }: SceneComponentProps) => { + return ; + }; +} diff --git a/public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts b/public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts new file mode 100644 index 00000000000..039d68f031e --- /dev/null +++ b/public/app/features/scenes/variables/variants/query/createQueryVariableRunner.ts @@ -0,0 +1,115 @@ +import { from, mergeMap, Observable, of } from 'rxjs'; + +import { + DataQuery, + DataQueryRequest, + DataSourceApi, + getDefaultTimeRange, + LoadingState, + PanelData, +} from '@grafana/data'; +import { runRequest } from 'app/features/query/state/runRequest'; +import { hasLegacyVariableSupport, hasStandardVariableSupport } from 'app/features/variables/guard'; + +import { QueryVariable } from './QueryVariable'; + +export interface RunnerArgs { + searchFilter?: string; + variable: QueryVariable; +} + +export interface QueryRunner { + getTarget: (variable: QueryVariable) => DataQuery; + runRequest: (args: RunnerArgs, request: DataQueryRequest) => Observable; +} + +class StandardQueryRunner implements QueryRunner { + public constructor(private datasource: DataSourceApi, private _runRequest = runRequest) {} + + public getTarget(variable: QueryVariable) { + if (hasStandardVariableSupport(this.datasource)) { + return this.datasource.variables.toDataQuery(variable.state.query); + } + + throw new Error("Couldn't create a target with supplied arguments."); + } + + public runRequest(_: RunnerArgs, request: DataQueryRequest) { + if (!hasStandardVariableSupport(this.datasource)) { + return getEmptyMetricFindValueObservable(); + } + + if (!this.datasource.variables.query) { + return this._runRequest(this.datasource, request); + } + + return this._runRequest(this.datasource, request, this.datasource.variables.query); + } +} + +class LegacyQueryRunner implements QueryRunner { + public constructor(private datasource: DataSourceApi) {} + + public getTarget(variable: QueryVariable) { + if (hasLegacyVariableSupport(this.datasource)) { + return variable.state.query; + } + + throw new Error("Couldn't create a target with supplied arguments."); + } + + public runRequest({ variable }: RunnerArgs, request: DataQueryRequest) { + if (!hasLegacyVariableSupport(this.datasource)) { + return getEmptyMetricFindValueObservable(); + } + + return from( + this.datasource.metricFindQuery(variable.state.query, { + ...request, + // variable is used by SQL common data source + variable: { + name: variable.state.name, + type: variable.state.type, + }, + // TODO: add support for search filter + // searchFilter + }) + ).pipe( + mergeMap((values) => { + if (!values || !values.length) { + return getEmptyMetricFindValueObservable(); + } + + const series: any = values; + return of({ series, state: LoadingState.Done, timeRange: request.range }); + }) + ); + } +} + +function getEmptyMetricFindValueObservable(): Observable { + return of({ state: LoadingState.Done, series: [], timeRange: getDefaultTimeRange() }); +} + +function createQueryVariableRunnerFactory(datasource: DataSourceApi): QueryRunner { + if (hasStandardVariableSupport(datasource)) { + return new StandardQueryRunner(datasource, runRequest); + } + + if (hasLegacyVariableSupport(datasource)) { + return new LegacyQueryRunner(datasource); + } + + // TODO: add support for legacy, cutom and datasource query runners + + throw new Error(`Couldn't create a query runner for datasource ${datasource.type}`); +} + +export let createQueryVariableRunner = createQueryVariableRunnerFactory; + +/** + * Use only in tests + */ +export function setCreateQueryVariableRunnerFactory(fn: (datasource: DataSourceApi) => QueryRunner) { + createQueryVariableRunner = fn; +} diff --git a/public/app/features/scenes/variables/variants/query/utils.ts b/public/app/features/scenes/variables/variants/query/utils.ts new file mode 100644 index 00000000000..6f720afff26 --- /dev/null +++ b/public/app/features/scenes/variables/variants/query/utils.ts @@ -0,0 +1,111 @@ +import { isNumber, sortBy, toLower, uniqBy } from 'lodash'; + +import { stringToJsRegex, VariableSort } from '@grafana/data'; + +import { VariableValueOption } from '../../types'; + +export const metricNamesToVariableValues = (variableRegEx: string, sort: VariableSort, metricNames: any[]) => { + let regex; + let options: VariableValueOption[] = []; + + if (variableRegEx) { + regex = stringToJsRegex(variableRegEx); + } + + for (let i = 0; i < metricNames.length; i++) { + const item = metricNames[i]; + let text = item.text === undefined || item.text === null ? item.value : item.text; + let value = item.value === undefined || item.value === null ? item.text : item.value; + + if (isNumber(value)) { + value = value.toString(); + } + + if (isNumber(text)) { + text = text.toString(); + } + + if (regex) { + const matches = getAllMatches(value, regex); + if (!matches.length) { + continue; + } + + const valueGroup = matches.find((m) => m.groups && m.groups.value); + const textGroup = matches.find((m) => m.groups && m.groups.text); + const firstMatch = matches.find((m) => m.length > 1); + const manyMatches = matches.length > 1 && firstMatch; + + if (valueGroup || textGroup) { + value = valueGroup?.groups?.value ?? textGroup?.groups?.text; + text = textGroup?.groups?.text ?? valueGroup?.groups?.value; + } else if (manyMatches) { + for (let j = 0; j < matches.length; j++) { + const match = matches[j]; + options.push({ label: match[1], value: match[1] }); + } + continue; + } else if (firstMatch) { + text = firstMatch[1]; + value = firstMatch[1]; + } + } + + options.push({ label: text, value: value }); + } + + options = uniqBy(options, 'value'); + return sortVariableValues(options, sort); +}; + +const getAllMatches = (str: string, regex: RegExp): RegExpExecArray[] => { + const results: RegExpExecArray[] = []; + let matches = null; + + regex.lastIndex = 0; + + do { + matches = regex.exec(str); + if (matches) { + results.push(matches); + } + } while (regex.global && matches && matches[0] !== '' && matches[0] !== undefined); + + return results; +}; + +export const sortVariableValues = (options: any[], sortOrder: VariableSort) => { + if (sortOrder === VariableSort.disabled) { + return options; + } + + const sortType = Math.ceil(sortOrder / 2); + const reverseSort = sortOrder % 2 === 0; + + if (sortType === 1) { + options = sortBy(options, 'text'); + } else if (sortType === 2) { + options = sortBy(options, (opt) => { + if (!opt.text) { + return -1; + } + + const matches = opt.text.match(/.*?(\d+).*/); + if (!matches || matches.length < 2) { + return -1; + } else { + return parseInt(matches[1], 10); + } + }); + } else if (sortType === 3) { + options = sortBy(options, (opt) => { + return toLower(opt.text); + }); + } + + if (reverseSort) { + options = options.reverse(); + } + + return options; +}; diff --git a/public/app/features/templating/LegacyVariableWrapper.ts b/public/app/features/templating/LegacyVariableWrapper.ts index a16d196d4a7..7ab0e2f9755 100644 --- a/public/app/features/templating/LegacyVariableWrapper.ts +++ b/public/app/features/templating/LegacyVariableWrapper.ts @@ -1,12 +1,14 @@ +import { VariableModel, VariableType } from '@grafana/schema'; + import { FormatVariable } from '../scenes/variables/interpolation/formatRegistry'; import { VariableValue } from '../scenes/variables/types'; import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE } from '../variables/constants'; export class LegacyVariableWrapper implements FormatVariable { - state: { name: string; value: VariableValue; text: VariableValue }; + state: { name: string; value: VariableValue; text: VariableValue; type: VariableType }; - constructor(name: string, value: VariableValue, text: VariableValue) { - this.state = { name, value, text }; + constructor(variable: VariableModel, value: VariableValue, text: VariableValue) { + this.state = { name: variable.name, value, text, type: variable.type }; } getValue(_fieldPath: string): VariableValue { @@ -40,11 +42,14 @@ let legacyVariableWrapper: LegacyVariableWrapper | undefined; /** * Reuses a single instance to avoid unnecessary memory allocations */ -export function getVariableWrapper(name: string, value: VariableValue, text: VariableValue) { +export function getVariableWrapper(variable: VariableModel, value: VariableValue, text: VariableValue) { + // TODO: provide more legacy variable properties, i.e. multi, includeAll that are used in custom interpolators, + // see Prometheus data source for example if (!legacyVariableWrapper) { - legacyVariableWrapper = new LegacyVariableWrapper(name, value, text); + legacyVariableWrapper = new LegacyVariableWrapper(variable, value, text); } else { - legacyVariableWrapper.state.name = name; + legacyVariableWrapper.state.name = variable.name; + legacyVariableWrapper.state.type = variable.type; legacyVariableWrapper.state.value = value; legacyVariableWrapper.state.text = text; } diff --git a/public/app/features/templating/template_srv.test.ts b/public/app/features/templating/template_srv.test.ts index d0f18300f7d..6b16204ca0e 100644 --- a/public/app/features/templating/template_srv.test.ts +++ b/public/app/features/templating/template_srv.test.ts @@ -5,6 +5,7 @@ import { silenceConsoleOutput } from '../../../test/core/utils/silenceConsoleOut import { initTemplateSrv } from '../../../test/helpers/initTemplateSrv'; import { mockDataSource, MockDataSourceSrv } from '../alerting/unified/mocks'; import { FormatRegistryID } from '../scenes/variables/interpolation/formatRegistry'; +import { TestVariable } from '../scenes/variables/variants/TestVariable'; import { VariableAdapter, variableAdapters } from '../variables/adapters'; import { createAdHocVariableAdapter } from '../variables/adhoc/adapter'; import { createQueryVariableAdapter } from '../variables/query/adapter'; @@ -17,6 +18,14 @@ variableAdapters.setInit(() => [ createAdHocVariableAdapter() as unknown as VariableAdapter, ]); +const interpolateMock = jest.fn(); + +jest.mock('../scenes/core/sceneGraph', () => ({ + sceneGraph: { + interpolate: (...args: any[]) => interpolateMock(...args), + }, +})); + describe('templateSrv', () => { silenceConsoleOutput(); let _templateSrv: any; @@ -810,4 +819,19 @@ describe('templateSrv', () => { expect(target).toBe('var-adhoc=value2'); }); }); + + describe('scenes compatibility', () => { + beforeEach(() => { + _templateSrv = initTemplateSrv(key, []); + }); + it('should use scene interpolator when scoped var provided', () => { + const variable = new TestVariable({}); + + _templateSrv.replace('test ${test}', { __sceneObject: { value: variable } }); + + expect(interpolateMock).toHaveBeenCalledTimes(1); + expect(interpolateMock.mock.calls[0][0]).toEqual(variable); + expect(interpolateMock.mock.calls[0][1]).toEqual('test ${test}'); + }); + }); }); diff --git a/public/app/features/templating/template_srv.ts b/public/app/features/templating/template_srv.ts index 2ed9730600a..1bed63eddd1 100644 --- a/public/app/features/templating/template_srv.ts +++ b/public/app/features/templating/template_srv.ts @@ -10,7 +10,10 @@ import { } from '@grafana/data'; import { getDataSourceSrv, setTemplateSrv, TemplateSrv as BaseTemplateSrv } from '@grafana/runtime'; +import { SceneObjectBase } from '../scenes/core/SceneObjectBase'; +import { sceneGraph } from '../scenes/core/sceneGraph'; import { formatRegistry, FormatRegistryID } from '../scenes/variables/interpolation/formatRegistry'; +import { CustomFormatterFn } from '../scenes/variables/interpolation/sceneInterpolator'; import { variableAdapters } from '../variables/adapters'; import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE } from '../variables/constants'; import { isAdHoc } from '../variables/guard'; @@ -166,7 +169,7 @@ export class TemplateSrv implements BaseTemplateSrv { formatItem = formatRegistry.get(FormatRegistryID.glob); } - const formatVariable = getVariableWrapper(variable.name, value, text ?? value); + const formatVariable = getVariableWrapper(variable, value, text ?? value); return formatItem.formatter(value, args, formatVariable); } @@ -276,6 +279,15 @@ export class TemplateSrv implements BaseTemplateSrv { } replace(target?: string, scopedVars?: ScopedVars, format?: string | Function): string { + if (scopedVars && scopedVars.__sceneObject && scopedVars.__sceneObject.value instanceof SceneObjectBase) { + return sceneGraph.interpolate( + scopedVars.__sceneObject.value, + target, + scopedVars, + format as string | CustomFormatterFn | undefined + ); + } + if (!target) { return target ?? ''; } diff --git a/public/test/setupTests.ts b/public/test/setupTests.ts index 31f322f1d4a..ac9da47b115 100644 --- a/public/test/setupTests.ts +++ b/public/test/setupTests.ts @@ -6,7 +6,7 @@ import { initReactI18next } from 'react-i18next'; import { matchers } from './matchers'; failOnConsole({ - shouldFailOnLog: process.env.CI ? true : false, + shouldFailOnLog: true, }); expect.extend(matchers);