[--Dashboard-- data source] Implement AdHoc filtering (#108011)
Behind a feature toggle: `dashboardDsAdHocFiltering`
This commit is contained in:
@@ -1054,4 +1054,8 @@ export interface FeatureToggles {
|
||||
* Enable dual reader for unified storage search
|
||||
*/
|
||||
unifiedStorageSearchDualReaderEnabled?: boolean;
|
||||
/**
|
||||
* Enables adhoc filtering support for the dashboard datasource
|
||||
*/
|
||||
dashboardDsAdHocFiltering?: boolean;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ const (
|
||||
grafanaAppPlatformSquad codeowner = "@grafana/grafana-app-platform-squad"
|
||||
grafanaDashboardsSquad codeowner = "@grafana/dashboards-squad"
|
||||
grafanaDatavizSquad codeowner = "@grafana/dataviz-squad"
|
||||
grafanaDataProSquad codeowner = "@grafana/datapro"
|
||||
grafanaFrontendPlatformSquad codeowner = "@grafana/grafana-frontend-platform"
|
||||
grafanaFrontendSearchNavOrganise codeowner = "@grafana/grafana-search-navigate-organise"
|
||||
grafanaBackendServicesSquad codeowner = "@grafana/grafana-backend-services-squad"
|
||||
|
||||
@@ -1821,6 +1821,13 @@ var (
|
||||
HideFromAdminPage: true,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "dashboardDsAdHocFiltering",
|
||||
Description: "Enables adhoc filtering support for the dashboard datasource",
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: grafanaDataProSquad,
|
||||
FrontendOnly: true,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -235,3 +235,4 @@ otelLogsFormatting,experimental,@grafana/observability-logs,false,false,true
|
||||
alertingNotificationHistory,experimental,@grafana/alerting-squad,false,false,false
|
||||
pluginAssetProvider,experimental,@grafana/plugins-platform-backend,false,true,false
|
||||
unifiedStorageSearchDualReaderEnabled,experimental,@grafana/search-and-storage,false,false,false
|
||||
dashboardDsAdHocFiltering,experimental,@grafana/datapro,false,false,true
|
||||
|
||||
|
@@ -950,4 +950,8 @@ const (
|
||||
// FlagUnifiedStorageSearchDualReaderEnabled
|
||||
// Enable dual reader for unified storage search
|
||||
FlagUnifiedStorageSearchDualReaderEnabled = "unifiedStorageSearchDualReaderEnabled"
|
||||
|
||||
// FlagDashboardDsAdHocFiltering
|
||||
// Enables adhoc filtering support for the dashboard datasource
|
||||
FlagDashboardDsAdHocFiltering = "dashboardDsAdHocFiltering"
|
||||
)
|
||||
|
||||
@@ -796,6 +796,19 @@
|
||||
"codeowner": "@grafana/grafana-app-platform-squad"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "dashboardDsAdHocFiltering",
|
||||
"resourceVersion": "1752487974435",
|
||||
"creationTimestamp": "2025-07-14T10:12:54Z"
|
||||
},
|
||||
"spec": {
|
||||
"description": "Enables adhoc filtering support for the dashboard datasource",
|
||||
"stage": "experimental",
|
||||
"codeowner": "@grafana/datapro",
|
||||
"frontend": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "dashboardNewLayouts",
|
||||
|
||||
+9
-1
@@ -4,6 +4,7 @@ import { DataSourceInstanceSettings, MetricFindValue, readCSV } from '@grafana/d
|
||||
import { selectors } from '@grafana/e2e-selectors';
|
||||
import { Trans, t } from '@grafana/i18n';
|
||||
import { EditorField } from '@grafana/plugin-ui';
|
||||
import { config } from '@grafana/runtime';
|
||||
import { DataSourceRef } from '@grafana/schema';
|
||||
import { Alert, CodeEditor, Field, Switch, Box } from '@grafana/ui';
|
||||
import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker';
|
||||
@@ -61,7 +62,14 @@ export function AdHocVariableForm({
|
||||
htmlFor="data-source-picker"
|
||||
tooltip={infoText}
|
||||
>
|
||||
<DataSourcePicker current={datasource} onChange={onDataSourceChange} width={30} variables={true} noDefault />
|
||||
<DataSourcePicker
|
||||
current={datasource}
|
||||
onChange={onDataSourceChange}
|
||||
width={30}
|
||||
variables={true}
|
||||
dashboard={config.featureToggles.dashboardDsAdHocFiltering}
|
||||
noDefault
|
||||
/>
|
||||
</EditorField>
|
||||
</Box>
|
||||
|
||||
|
||||
@@ -7,15 +7,19 @@ import {
|
||||
getDefaultTimeRange,
|
||||
LoadingState,
|
||||
standardTransformersRegistry,
|
||||
FieldType,
|
||||
DataFrame,
|
||||
AdHocVariableFilter,
|
||||
} from '@grafana/data';
|
||||
import { getPanelPlugin } from '@grafana/data/test';
|
||||
import { setPluginImportUtils } from '@grafana/runtime';
|
||||
import { setPluginImportUtils, config } from '@grafana/runtime';
|
||||
import {
|
||||
SafeSerializableSceneObject,
|
||||
SceneDataNode,
|
||||
SceneDataTransformer,
|
||||
SceneFlexItem,
|
||||
SceneFlexLayout,
|
||||
SceneObject,
|
||||
VizPanel,
|
||||
} from '@grafana/scenes';
|
||||
import { getVizPanelKeyForPanelId } from 'app/features/dashboard-scene/utils/utils';
|
||||
@@ -131,6 +135,441 @@ describe('DashboardDatasource', () => {
|
||||
// on further emissions the result should be the unmodified original dataframe
|
||||
expect(rsp!.data[0].fields[0].state).toEqual({});
|
||||
});
|
||||
|
||||
describe('AdHoc Filtering', () => {
|
||||
// Shared test utilities
|
||||
interface TestField {
|
||||
name: string;
|
||||
type: FieldType;
|
||||
values: unknown[];
|
||||
}
|
||||
|
||||
function createTestFrame(fields: TestField[]) {
|
||||
return {
|
||||
name: 'TestData',
|
||||
fields: fields.map((field) => ({
|
||||
name: field.name,
|
||||
type: field.type,
|
||||
values: field.values,
|
||||
config: {},
|
||||
state: {},
|
||||
})),
|
||||
length: fields[0]?.values.length || 0,
|
||||
refId: 'A',
|
||||
};
|
||||
}
|
||||
|
||||
function createQueryRequest(filters: AdHocVariableFilter[], scene: SceneObject) {
|
||||
return {
|
||||
timezone: 'utc',
|
||||
targets: [{ refId: 'A', panelId: 1 }],
|
||||
requestId: '',
|
||||
interval: '',
|
||||
intervalMs: 0,
|
||||
range: getDefaultTimeRange(),
|
||||
scopedVars: {
|
||||
__sceneObject: new SafeSerializableSceneObject(scene),
|
||||
},
|
||||
app: '',
|
||||
startTime: 0,
|
||||
filters: filters,
|
||||
};
|
||||
}
|
||||
|
||||
// Test AdHoc filtering via the Public API first, to ensure Integration
|
||||
describe('Integration (Public API)', () => {
|
||||
const originalToggleValue = config.featureToggles.dashboardDsAdHocFiltering;
|
||||
|
||||
beforeEach(() => {
|
||||
config.featureToggles.dashboardDsAdHocFiltering = true;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
config.featureToggles.dashboardDsAdHocFiltering = originalToggleValue;
|
||||
});
|
||||
|
||||
it('should apply basic filtering end-to-end through public query method', async () => {
|
||||
const testFrame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const scene = new SceneFlexLayout({
|
||||
children: [
|
||||
new SceneFlexItem({
|
||||
body: new VizPanel({
|
||||
key: getVizPanelKeyForPanelId(1),
|
||||
$data: new SceneDataNode({
|
||||
data: {
|
||||
series: [testFrame],
|
||||
state: LoadingState.Done,
|
||||
timeRange: getDefaultTimeRange(),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const ds = new DashboardDatasource({} as DataSourceInstanceSettings);
|
||||
const filters: AdHocVariableFilter[] = [{ key: 'name', operator: '=', value: 'John' }];
|
||||
|
||||
const observable = ds.query(createQueryRequest(filters, scene));
|
||||
|
||||
let result: DataQueryResponse | undefined;
|
||||
observable.subscribe({ next: (data) => (result = data) });
|
||||
|
||||
expect(result?.data[0].fields[0].values).toEqual(['John']);
|
||||
expect(result?.data[0].fields[1].values).toEqual([25]);
|
||||
expect(result?.data[0].length).toBe(1);
|
||||
});
|
||||
|
||||
it('should respect feature toggle and not filter when disabled', async () => {
|
||||
// Temporarily disable the feature toggle for this test
|
||||
config.featureToggles.dashboardDsAdHocFiltering = false;
|
||||
|
||||
const testFrame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const scene = new SceneFlexLayout({
|
||||
children: [
|
||||
new SceneFlexItem({
|
||||
body: new VizPanel({
|
||||
key: getVizPanelKeyForPanelId(1),
|
||||
$data: new SceneDataNode({
|
||||
data: {
|
||||
series: [testFrame],
|
||||
state: LoadingState.Done,
|
||||
timeRange: getDefaultTimeRange(),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const ds = new DashboardDatasource({} as DataSourceInstanceSettings);
|
||||
const filters: AdHocVariableFilter[] = [{ key: 'name', operator: '=', value: 'John' }];
|
||||
|
||||
const observable = ds.query(createQueryRequest(filters, scene));
|
||||
|
||||
let result: DataQueryResponse | undefined;
|
||||
observable.subscribe({ next: (data) => (result = data) });
|
||||
|
||||
// Should return unfiltered data since feature toggle is disabled
|
||||
expect(result?.data[0].fields[0].values).toEqual(['John', 'Jane', 'Bob']);
|
||||
expect(result?.data[0].fields[1].values).toEqual([25, 30, 35]);
|
||||
expect(result?.data[0].length).toBe(3);
|
||||
});
|
||||
|
||||
it('should apply multiple filters with AND logic through public API', async () => {
|
||||
const testFrame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'status', type: FieldType.string, values: ['active', 'active', 'inactive'] },
|
||||
]);
|
||||
|
||||
const scene = new SceneFlexLayout({
|
||||
children: [
|
||||
new SceneFlexItem({
|
||||
body: new VizPanel({
|
||||
key: getVizPanelKeyForPanelId(1),
|
||||
$data: new SceneDataNode({
|
||||
data: {
|
||||
series: [testFrame],
|
||||
state: LoadingState.Done,
|
||||
timeRange: getDefaultTimeRange(),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const ds = new DashboardDatasource({} as DataSourceInstanceSettings);
|
||||
const filters: AdHocVariableFilter[] = [
|
||||
{ key: 'name', operator: '!=', value: 'John' },
|
||||
{ key: 'status', operator: '=', value: 'active' },
|
||||
];
|
||||
|
||||
const observable = ds.query(createQueryRequest(filters, scene));
|
||||
|
||||
let result: DataQueryResponse | undefined;
|
||||
observable.subscribe({ next: (data) => (result = data) });
|
||||
|
||||
expect(result?.data[0].fields[0].values).toEqual(['Jane']);
|
||||
expect(result?.data[0].fields[1].values).toEqual(['active']);
|
||||
expect(result?.data[0].length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// Now test AdHoc filtering with unit-tests to test all aspects of behaviour.
|
||||
// Here we test a private method, which allows for smaller, simpler,
|
||||
// faster tests.
|
||||
// This is the bottom of the 'Test Pyramid'
|
||||
describe('Algorithm (but by testing a Private Method)', () => {
|
||||
const ds = new DashboardDatasource({} as DataSourceInstanceSettings);
|
||||
|
||||
// Type-safe interface for accessing private methods in tests
|
||||
interface DashboardDatasourceWithPrivateMethods {
|
||||
applyAdHocFilters: (frame: DataFrame, filters: AdHocVariableFilter[]) => DataFrame;
|
||||
}
|
||||
|
||||
it('should apply equality filter correctly', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '=', value: 'John' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['John']);
|
||||
expect(result.fields[1].values).toEqual([25]);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should apply not-equal filter correctly', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '!=', value: 'John' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Jane', 'Bob']);
|
||||
expect(result.fields[1].values).toEqual([30, 35]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should apply multiple filters with AND logic', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'status', type: FieldType.string, values: ['active', 'active', 'inactive'] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '!=', value: 'John' },
|
||||
{ key: 'status', operator: '=', value: 'active' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Jane']);
|
||||
expect(result.fields[1].values).toEqual(['active']);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle null values correctly', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', null, 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '!=', value: 'John' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual([null, 'Bob']);
|
||||
expect(result.fields[1].values).toEqual([30, 35]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should apply equality filter on numeric fields', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'age', operator: '=', value: '30' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Jane']);
|
||||
expect(result.fields[1].values).toEqual([30]);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should apply not-equal filter on numeric fields', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'age', operator: '!=', value: '30' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['John', 'Bob']);
|
||||
expect(result.fields[1].values).toEqual([25, 35]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should handle numeric fields with null values', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, null, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'age', operator: '!=', value: '25' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Jane', 'Bob']);
|
||||
expect(result.fields[1].values).toEqual([null, 35]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should handle mixed string and numeric filtering', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob', 'Alice'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 25, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '!=', value: 'Bob' },
|
||||
{ key: 'age', operator: '=', value: '25' },
|
||||
]);
|
||||
|
||||
// Should match: name != 'Bob' AND age = 25
|
||||
// John: !Bob + 25 ✓
|
||||
// Jane: !Bob + 30 ✗
|
||||
// Bob: Bob + 25 ✗
|
||||
// Alice: !Bob + 35 ✗
|
||||
expect(result.fields[0].values).toEqual(['John']);
|
||||
expect(result.fields[1].values).toEqual([25]);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle floating point numbers correctly', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['Alice', 'Bob', 'Charlie', 'Diana'] },
|
||||
{ name: 'score', type: FieldType.number, values: [95.5, 87.25, 95.5, 92.0] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'score', operator: '=', value: '95.5' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Alice', 'Charlie']);
|
||||
expect(result.fields[1].values).toEqual([95.5, 95.5]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
|
||||
it('should handle floating point numbers with != operator', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['Alice', 'Bob', 'Charlie'] },
|
||||
{ name: 'temperature', type: FieldType.number, values: [98.6, 99.1, 98.6] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'temperature', operator: '!=', value: '98.6' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual(['Bob']);
|
||||
expect(result.fields[1].values).toEqual([99.1]);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle precision edge cases with floats', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['Test1', 'Test2', 'Test3'] },
|
||||
{ name: 'value', type: FieldType.number, values: [0.1 + 0.2, 0.3, 1.0000001] },
|
||||
]);
|
||||
|
||||
// Note: 0.1 + 0.2 !== 0.3 in JavaScript due to floating point precision
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'value', operator: '=', value: '0.3' },
|
||||
]);
|
||||
|
||||
// Should only match the exact 0.3, not the computed 0.1 + 0.2
|
||||
expect(result.fields[0].values).toEqual(['Test2']);
|
||||
expect(result.fields[1].values).toEqual([0.3]);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle empty data frames', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: [] },
|
||||
{ name: 'age', type: FieldType.number, values: [] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'name', operator: '=', value: 'John' },
|
||||
]);
|
||||
|
||||
expect(result.fields[0].values).toEqual([]);
|
||||
expect(result.fields[1].values).toEqual([]);
|
||||
expect(result.length).toBe(0);
|
||||
});
|
||||
|
||||
it.skip('should handle remaining operators', () => {
|
||||
// Not yet implemented, so we explicitly don't specify any behaviour for this
|
||||
});
|
||||
|
||||
it.skip('should handle remaining field types (eg. date)', () => {
|
||||
// Not yet implemented, so we explicitly don't specify any behaviour for this
|
||||
});
|
||||
|
||||
it('should handle filters on missing fields with = operator', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'missing_field', operator: '=', value: 'test' },
|
||||
]);
|
||||
|
||||
// Should return empty result since field doesn't exist
|
||||
expect(result.fields[0].values).toEqual([]);
|
||||
expect(result.fields[1].values).toEqual([]);
|
||||
expect(result.length).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle filters on missing fields with != operator', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Bob'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'missing_field', operator: '!=', value: 'test' },
|
||||
]);
|
||||
|
||||
// Should return all rows since field doesn't exist (all rows are "not equal")
|
||||
expect(result.fields[0].values).toEqual(['John', 'Jane', 'Bob']);
|
||||
expect(result.fields[1].values).toEqual([25, 30, 35]);
|
||||
expect(result.length).toBe(3);
|
||||
});
|
||||
|
||||
it('should handle complex filtering scenario', () => {
|
||||
const frame = createTestFrame([
|
||||
{ name: 'name', type: FieldType.string, values: ['John', 'Jane', 'Admin', 'Bob'] },
|
||||
{ name: 'status', type: FieldType.string, values: ['active', 'inactive', 'active', 'active'] },
|
||||
{ name: 'age', type: FieldType.number, values: [25, 30, 35, 40] },
|
||||
]);
|
||||
|
||||
const result = (ds as unknown as DashboardDatasourceWithPrivateMethods).applyAdHocFilters(frame, [
|
||||
{ key: 'status', operator: '=', value: 'active' },
|
||||
{ key: 'name', operator: '!=', value: 'Admin' },
|
||||
{ key: 'missing_field', operator: '!=', value: 'ignored' }, // Should be ignored
|
||||
]);
|
||||
|
||||
// Should match: status=active AND name!=Admin
|
||||
// John: active + !Admin ✓
|
||||
// Jane: inactive + !Admin ✗
|
||||
// Admin: active + Admin ✗
|
||||
// Bob: active + !Admin ✓
|
||||
expect(result.fields[0].values).toEqual(['John', 'Bob']);
|
||||
expect(result.fields[1].values).toEqual(['active', 'active']);
|
||||
expect(result.fields[2].values).toEqual([25, 40]);
|
||||
expect(result.length).toBe(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function setup(query: DashboardQuery, requestId?: string) {
|
||||
@@ -140,7 +579,6 @@ function setup(query: DashboardQuery, requestId?: string) {
|
||||
series: [arrayToDataFrame([1, 2, 3])],
|
||||
state: LoadingState.Done,
|
||||
timeRange: getDefaultTimeRange(),
|
||||
structureRev: 11,
|
||||
},
|
||||
}),
|
||||
transformations: [{ id: 'reduce', options: {} }],
|
||||
|
||||
@@ -12,7 +12,13 @@ import {
|
||||
DataFrame,
|
||||
LoadingState,
|
||||
Field,
|
||||
FieldType,
|
||||
AdHocVariableFilter,
|
||||
MetricFindValue,
|
||||
getValueMatcher,
|
||||
ValueMatcherID,
|
||||
} from '@grafana/data';
|
||||
import { config } from '@grafana/runtime';
|
||||
import { SceneDataProvider, SceneDataTransformer, SceneObject } from '@grafana/scenes';
|
||||
import {
|
||||
activateSceneObjectAndParentTree,
|
||||
@@ -71,6 +77,9 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
return of({ data: [] });
|
||||
}
|
||||
|
||||
// Extract AdHoc filters from the request
|
||||
const adHocFilters = options.filters || [];
|
||||
|
||||
return defer(() => {
|
||||
if (!sourceDataProvider!.isActive && sourceDataProvider?.setContainerWidth) {
|
||||
sourceDataProvider?.setContainerWidth(500);
|
||||
@@ -82,7 +91,7 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
debounceTime(50),
|
||||
map((result) => {
|
||||
return {
|
||||
data: this.getDataFramesForQueryTopic(result.data, query),
|
||||
data: this.getDataFramesForQueryTopic(result.data, query, adHocFilters),
|
||||
state: result.data.state,
|
||||
errors: result.data.errors,
|
||||
error: result.data.error,
|
||||
@@ -95,7 +104,11 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
});
|
||||
}
|
||||
|
||||
private getDataFramesForQueryTopic(data: PanelData, query: DashboardQuery): DataFrame[] {
|
||||
private getDataFramesForQueryTopic(
|
||||
data: PanelData,
|
||||
query: DashboardQuery,
|
||||
filters: AdHocVariableFilter[]
|
||||
): DataFrame[] {
|
||||
const annotations = data.annotations ?? [];
|
||||
if (query.topic === DataTopic.Annotations) {
|
||||
return annotations.map((frame) => ({
|
||||
@@ -111,6 +124,13 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
...s,
|
||||
fields: s.fields.map((field: Field) => ({
|
||||
...field,
|
||||
config: {
|
||||
...field.config,
|
||||
// Enable AdHoc filtering for string and numeric fields only when feature toggle is enabled
|
||||
filterable: config.featureToggles.dashboardDsAdHocFiltering
|
||||
? field.type === FieldType.string || field.type === FieldType.number
|
||||
: field.config.filterable,
|
||||
},
|
||||
state: {
|
||||
...field.state,
|
||||
},
|
||||
@@ -118,10 +138,149 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
};
|
||||
});
|
||||
|
||||
return [...series, ...annotations];
|
||||
if (!config.featureToggles.dashboardDsAdHocFiltering || filters.length === 0) {
|
||||
return [...series, ...annotations];
|
||||
}
|
||||
|
||||
// Apply AdHoc filters to series data
|
||||
const filteredSeries = series.map((frame) => this.applyAdHocFilters(frame, filters));
|
||||
return [...filteredSeries, ...annotations];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply AdHoc filters to a DataFrame
|
||||
* Optimized version with pre-computed field indices and value matchers for better performance
|
||||
*/
|
||||
private applyAdHocFilters(frame: DataFrame, filters: AdHocVariableFilter[]): DataFrame {
|
||||
if (filters.length === 0 || frame.length === 0) {
|
||||
return frame;
|
||||
}
|
||||
|
||||
// Pre-compute field indices and value matchers for better performance
|
||||
const filterFieldIndices = filters
|
||||
.map((filter) => {
|
||||
const fieldIndex = frame.fields.findIndex((f) => f.name === filter.key);
|
||||
return { filter, fieldIndex, matcher: this.createValueMatcher(filter, fieldIndex, frame) };
|
||||
})
|
||||
.filter(({ filter, fieldIndex, matcher }) => {
|
||||
// If field is not present:
|
||||
// - Keep filters with '=' operator (will always be false - reject rows)
|
||||
// - Remove filters with '!=' operator (will always be true - no effect)
|
||||
if (fieldIndex === -1) {
|
||||
return filter.operator === '=';
|
||||
}
|
||||
// Only keep filters with valid matchers
|
||||
return matcher !== null;
|
||||
});
|
||||
|
||||
// If no filters remain after optimization, return original frame
|
||||
if (filterFieldIndices.length === 0) {
|
||||
return frame;
|
||||
}
|
||||
|
||||
// Short-circuit: if any filter has '=' operator with missing field, reject all rows
|
||||
const hasImpossibleFilter = filterFieldIndices.some(({ fieldIndex }) => fieldIndex === -1);
|
||||
if (hasImpossibleFilter) {
|
||||
return this.reconstructDataFrame(frame);
|
||||
}
|
||||
|
||||
const matchingRows = new Set<number>();
|
||||
|
||||
// Check each row to see if it matches all filters (AND logic)
|
||||
for (let rowIndex = 0; rowIndex < frame.length; rowIndex++) {
|
||||
const rowMatches = filterFieldIndices.every(({ matcher, fieldIndex }) => {
|
||||
const field = frame.fields[fieldIndex];
|
||||
|
||||
// Use Grafana's value matcher system
|
||||
return matcher?.(rowIndex, field, frame, [frame]) ?? false;
|
||||
});
|
||||
|
||||
if (rowMatches) {
|
||||
matchingRows.add(rowIndex);
|
||||
}
|
||||
}
|
||||
|
||||
// Early return if no filtering occurred
|
||||
if (matchingRows.size === frame.length) {
|
||||
return frame;
|
||||
}
|
||||
|
||||
return this.reconstructDataFrame(frame, matchingRows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a value matcher from an AdHoc filter
|
||||
*/
|
||||
private createValueMatcher(filter: AdHocVariableFilter, fieldIndex: number, frame: DataFrame) {
|
||||
// Return null for missing fields - they are handled separately
|
||||
if (fieldIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const field = frame.fields[fieldIndex];
|
||||
|
||||
// Only support string and numeric fields when feature toggle is enabled
|
||||
if (config.featureToggles.dashboardDsAdHocFiltering) {
|
||||
if (field.type !== FieldType.string && field.type !== FieldType.number) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Map operator to matcher ID
|
||||
let matcherId: ValueMatcherID;
|
||||
switch (filter.operator) {
|
||||
case '=':
|
||||
matcherId = ValueMatcherID.equal;
|
||||
break;
|
||||
case '!=':
|
||||
matcherId = ValueMatcherID.notEqual;
|
||||
break;
|
||||
default:
|
||||
return null; // Unknown operator
|
||||
}
|
||||
|
||||
try {
|
||||
return getValueMatcher({
|
||||
id: matcherId,
|
||||
options: { value: filter.value },
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn('Failed to create value matcher for filter:', filter, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct DataFrame with only matching rows
|
||||
* Optimized to avoid repeated array operations
|
||||
*/
|
||||
private reconstructDataFrame(frame: DataFrame, matchingRows?: Set<number>): DataFrame {
|
||||
// Default to empty set if no matching rows provided (reject all rows)
|
||||
const rows = matchingRows ?? new Set<number>();
|
||||
|
||||
const fields: Field[] = frame.fields.map((field) => {
|
||||
// Pre-allocate array and use direct assignment for better performance with large datasets
|
||||
const newValues = new Array(rows.size);
|
||||
let i = 0;
|
||||
for (const rowIndex of rows) {
|
||||
newValues[i++] = field.values[rowIndex];
|
||||
}
|
||||
|
||||
return {
|
||||
...field,
|
||||
values: newValues,
|
||||
state: {}, // Clean the state as it's being recalculated
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
...frame,
|
||||
fields: fields,
|
||||
length: rows.size,
|
||||
};
|
||||
}
|
||||
|
||||
private findSourcePanel(scene: SceneObject, panelId: number) {
|
||||
// We're trying to find the original panel, not a cloned one, since `panelId` alone cannot resolve clones
|
||||
return findOriginalVizPanelByKey(scene, getVizPanelKeyForPanelId(panelId));
|
||||
@@ -169,4 +328,10 @@ export class DashboardDatasource extends DataSourceApi<DashboardQuery> {
|
||||
testDatasource(): Promise<TestDataSourceResponse> {
|
||||
return Promise.resolve({ message: '', status: '' });
|
||||
}
|
||||
|
||||
getTagKeys(): Promise<MetricFindValue[]> {
|
||||
// Stub implementation to indicate AdHoc filter support
|
||||
// Full implementation will be added in future PRs
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user