Transformations: Add support for dashboard variable in limit, sort by, filter by value, heatmap and histogram (#75372)

* variables for filterforvalue

* use datalinkinput for basic matcher

* fix user select issue

* heatmap transformation variable interpolation

* clean code

* interpolate sort by

* add options interpolation in histogram transformation

* interpolation for limit

* Add suggestions UI to Filter by data value Transformation

Co-authored-by: oscarkilhed <oscar.kilhed@grafana.com>

* add validation for number/variable fields

* Add variables to add field from calculation

* Add validator to limit transformation

* Refactor validator

* Refactor suggestionInput styles

* Add variable support in heatmap calculate options to be in sync with tranform

* Refactor SuggestionsInput

* Fix histogram, limit and filter by value matchers

* clean up weird state ref

* Only interpolate when the feature toggle is set

* Add feature toggle to ui

* Fix number of variable test

* Fix issue with characters typed after opening suggestions still remains after selecting a suggestion

* Clean up from review

* Add more tests for numberOrVariableValidator

---------

Co-authored-by: Victor Marin <victor.marin@grafana.com>

packages/grafana-data/src/transformations/matchers/valueMatchers/numericMatchers.ts

@@ -4,7 +4,7 @@ import { ValueMatcherID } from '../ids';
 
 import { BasicValueMatcherOptions } from './types';
 
-const isGreaterValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>> = {
+const isGreaterValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions> = {
   id: ValueMatcherID.greater,
   name: 'Is greater',
   description: 'Match when field value is greater than option.',
@@ -24,7 +24,7 @@ const isGreaterValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>>
   getDefaultOptions: () => ({ value: 0 }),
 };
 
-const isGreaterOrEqualValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>> = {
+const isGreaterOrEqualValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions> = {
   id: ValueMatcherID.greaterOrEqual,
   name: 'Is greater or equal',
   description: 'Match when field value is greater than or equal to option.',
@@ -44,7 +44,7 @@ const isGreaterOrEqualValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<nu
   getDefaultOptions: () => ({ value: 0 }),
 };
 
-const isLowerValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>> = {
+const isLowerValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions> = {
   id: ValueMatcherID.lower,
   name: 'Is lower',
   description: 'Match when field value is lower than option.',
@@ -64,7 +64,7 @@ const isLowerValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>> =
   getDefaultOptions: () => ({ value: 0 }),
 };
 
-const isLowerOrEqualValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions<number>> = {
+const isLowerOrEqualValueMatcher: ValueMatcherInfo<BasicValueMatcherOptions> = {
   id: ValueMatcherID.lowerOrEqual,
   name: 'Is lower or equal',
   description: 'Match when field value is lower or equal than option.',

packages/grafana-data/src/transformations/matchers/valueMatchers/rangeMatchers.ts

@@ -4,7 +4,7 @@ import { ValueMatcherID } from '../ids';
 
 import { RangeValueMatcherOptions } from './types';
 
-const isBetweenValueMatcher: ValueMatcherInfo<RangeValueMatcherOptions<number>> = {
+const isBetweenValueMatcher: ValueMatcherInfo<RangeValueMatcherOptions> = {
   id: ValueMatcherID.between,
   name: 'Is between',
   description: 'Match when field value is between given option values.',

packages/grafana-data/src/transformations/transformers/filterByValue.test.ts

@@ -13,6 +13,18 @@ import {
 } from './filterByValue';
 import { DataTransformerID } from './ids';
 
+let transformationSupport = false;
+
+jest.mock('./utils', () => {
+  const actual = jest.requireActual('./utils');
+  return {
+    ...actual,
+    transformationsVariableSupport: () => {
+      return transformationSupport;
+    },
+  };
+});
+
 const seriesAWithSingleField = toDataFrame({
   name: 'A',
   length: 7,
@@ -109,6 +121,93 @@ describe('FilterByValue transformer', () => {
     });
   });
 
+  it('should interpolate dashboard variables', async () => {
+    transformationSupport = true;
+
+    const lower: MatcherConfig<BasicValueMatcherOptions<string | number>> = {
+      id: ValueMatcherID.lower,
+      options: { value: 'thiswillinterpolateto6' },
+    };
+
+    const cfg: DataTransformerConfig<FilterByValueTransformerOptions> = {
+      id: DataTransformerID.filterByValue,
+      options: {
+        type: FilterByValueType.exclude,
+        match: FilterByValueMatch.all,
+        filters: [
+          {
+            fieldName: 'numbers',
+            config: lower,
+          },
+        ],
+      },
+    };
+
+    const ctxmock = { interpolate: jest.fn(() => '6') };
+
+    await expect(transformDataFrame([cfg], [seriesAWithSingleField], ctxmock)).toEmitValuesWith((received) => {
+      const processed = received[0];
+
+      expect(processed.length).toEqual(1);
+      expect(processed[0].fields).toEqual([
+        {
+          name: 'time',
+          type: FieldType.time,
+          values: [6000, 7000],
+          state: {},
+        },
+        {
+          name: 'numbers',
+          type: FieldType.number,
+          values: [6, 7],
+          state: {},
+        },
+      ]);
+    });
+    transformationSupport = false;
+  });
+
+  it('should not interpolate dashboard variables when feature toggle is off', async () => {
+    const lower: MatcherConfig<BasicValueMatcherOptions<number | string>> = {
+      id: ValueMatcherID.lower,
+      options: { value: 'notinterpolating' },
+    };
+
+    const cfg: DataTransformerConfig<FilterByValueTransformerOptions> = {
+      id: DataTransformerID.filterByValue,
+      options: {
+        type: FilterByValueType.exclude,
+        match: FilterByValueMatch.all,
+        filters: [
+          {
+            fieldName: 'numbers',
+            config: lower,
+          },
+        ],
+      },
+    };
+
+    await expect(transformDataFrame([cfg], [seriesAWithSingleField])).toEmitValuesWith((received) => {
+      const processed = received[0];
+
+      expect(processed.length).toEqual(1);
+      expect(processed[0].fields).toEqual([
+        {
+          name: 'time',
+          type: FieldType.time,
+          values: [1000, 2000, 3000, 4000, 5000, 6000, 7000],
+          state: {},
+        },
+        {
+          name: 'numbers',
+          type: FieldType.number,
+          values: [1, 2, 3, 4, 5, 6, 7],
+          state: {},
+        },
+      ]);
+    });
+  });
+
   it('should match any condition', async () => {
     const lowerOrEqual: MatcherConfig<BasicValueMatcherOptions<number>> = {
       id: ValueMatcherID.lowerOrEqual,

packages/grafana-data/src/transformations/transformers/filterByValue.ts

@@ -4,9 +4,11 @@ import { getFieldDisplayName } from '../../field/fieldState';
 import { DataFrame, Field } from '../../types/dataFrame';
 import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
 import { getValueMatcher } from '../matchers';
+import { ValueMatcherID } from '../matchers/ids';
 
 import { DataTransformerID } from './ids';
 import { noopTransformer } from './noop';
+import { transformationsVariableSupport } from './utils';
 
 export enum FilterByValueType {
   exclude = 'exclude',
@@ -48,6 +50,46 @@ export const filterByValueTransformer: DataTransformerInfo<FilterByValueTransfor
       return source.pipe(noopTransformer.operator({}, ctx));
     }
 
+    const interpolatedFilters: FilterByValueFilter[] = [];
+
+    if (transformationsVariableSupport()) {
+      interpolatedFilters.push(
+        ...filters.map((filter) => {
+          if (filter.config.id === ValueMatcherID.between) {
+            const interpolatedFrom = ctx.interpolate(filter.config.options.from);
+            const interpolatedTo = ctx.interpolate(filter.config.options.to);
+
+            const newFilter = {
+              ...filter,
+              config: {
+                ...filter.config,
+                options: {
+                  ...filter.config.options,
+                  to: interpolatedTo,
+                  from: interpolatedFrom,
+                },
+              },
+            };
+
+            return newFilter;
+          } else if (filter.config.id === ValueMatcherID.regex) {
+            // Due to colliding syntaxes, interpolating regex filters will cause issues.
+            return filter;
+          } else if (filter.config.options.value) {
+            const interpolatedValue = ctx.interpolate(filter.config.options.value);
+            const newFilter = {
+              ...filter,
+              config: { ...filter.config, options: { ...filter.config.options, value: interpolatedValue } },
+            };
+            newFilter.config.options.value! = interpolatedValue;
+            return newFilter;
+          }
+
+          return filter;
+        })
+      );
+    }
+
     return source.pipe(
       map((data) => {
         if (!Array.isArray(data) || data.length === 0) {
@@ -58,7 +100,13 @@ export const filterByValueTransformer: DataTransformerInfo<FilterByValueTransfor
 
         for (const frame of data) {
           const fieldIndexByName = groupFieldIndexByName(frame, data);
-          const matchers = createFilterValueMatchers(filters, fieldIndexByName);
+
+          let matchers;
+          if (transformationsVariableSupport()) {
+            matchers = createFilterValueMatchers(interpolatedFilters, fieldIndexByName);
+          } else {
+            matchers = createFilterValueMatchers(filters, fieldIndexByName);
+          }
 
           for (let index = 0; index < frame.length; index++) {
             if (rows.has(index)) {

packages/grafana-data/src/transformations/transformers/histogram.ts

@@ -2,12 +2,13 @@ import { map } from 'rxjs/operators';
 
 import { getDisplayProcessor } from '../../field';
 import { createTheme, GrafanaTheme2 } from '../../themes';
-import { DataFrameType, SynchronousDataTransformerInfo } from '../../types';
+import { DataFrameType, DataTransformContext, SynchronousDataTransformerInfo } from '../../types';
 import { DataFrame, Field, FieldConfig, FieldType } from '../../types/dataFrame';
 import { roundDecimals } from '../../utils';
 
 import { DataTransformerID } from './ids';
 import { AlignedData, join } from './joinDataFrames';
+import { transformationsVariableSupport } from './utils';
 
 /**
  * @internal
@@ -40,6 +41,12 @@ export const histogramBucketSizes = [
 const histFilter = [null];
 const histSort = (a: number, b: number) => a - b;
 
+export interface HistogramTransformerInputs {
+  bucketSize?: string | number;
+  bucketOffset?: string | number;
+  combine?: boolean;
+}
+
 /**
  * @alpha
  */
@@ -74,7 +81,7 @@ export const histogramFieldInfo = {
 /**
  * @alpha
  */
-export const histogramTransformer: SynchronousDataTransformerInfo<HistogramTransformerOptions> = {
+export const histogramTransformer: SynchronousDataTransformerInfo<HistogramTransformerInputs> = {
   id: DataTransformerID.histogram,
   name: 'Histogram',
   description: 'Calculate a histogram from input data.',
@@ -85,11 +92,51 @@ export const histogramTransformer: SynchronousDataTransformerInfo<HistogramTrans
   operator: (options, ctx) => (source) =>
     source.pipe(map((data) => histogramTransformer.transformer(options, ctx)(data))),
 
-  transformer: (options: HistogramTransformerOptions) => (data: DataFrame[]) => {
+  transformer: (options: HistogramTransformerInputs, ctx: DataTransformContext) => (data: DataFrame[]) => {
     if (!Array.isArray(data) || data.length === 0) {
       return data;
     }
-    const hist = buildHistogram(data, options);
+
+    let bucketSize,
+      bucketOffset: number | undefined = undefined;
+
+    if (options.bucketSize) {
+      if (transformationsVariableSupport()) {
+        options.bucketSize = ctx.interpolate(options.bucketSize.toString());
+      }
+      if (typeof options.bucketSize === 'string') {
+        bucketSize = parseFloat(options.bucketSize);
+      } else {
+        bucketSize = options.bucketSize;
+      }
+
+      if (isNaN(bucketSize)) {
+        bucketSize = undefined;
+      }
+    }
+
+    if (options.bucketOffset) {
+      if (transformationsVariableSupport()) {
+        options.bucketOffset = ctx.interpolate(options.bucketOffset.toString());
+      }
+      if (typeof options.bucketOffset === 'string') {
+        bucketOffset = parseFloat(options.bucketOffset);
+      } else {
+        bucketOffset = options.bucketOffset;
+      }
+
+      if (isNaN(bucketOffset)) {
+        bucketOffset = undefined;
+      }
+    }
+
+    const interpolatedOptions: HistogramTransformerOptions = {
+      bucketSize: bucketSize,
+      bucketOffset: bucketOffset,
+      combine: options.combine,
+    };
+
+    const hist = buildHistogram(data, interpolatedOptions);
     if (hist == null) {
       return [];
     }

packages/grafana-data/src/transformations/transformers/limit.ts

@@ -3,9 +3,10 @@ import { map } from 'rxjs/operators';
 import { DataTransformerInfo } from '../../types';
 
 import { DataTransformerID } from './ids';
+import { transformationsVariableSupport } from './utils';
 
 export interface LimitTransformerOptions {
-  limitField?: number;
+  limitField?: number | string;
 }
 
 const DEFAULT_LIMIT_FIELD = 10;
@@ -18,21 +19,32 @@ export const limitTransformer: DataTransformerInfo<LimitTransformerOptions> = {
     limitField: DEFAULT_LIMIT_FIELD,
   },
 
-  operator: (options) => (source) =>
+  operator: (options, ctx) => (source) =>
     source.pipe(
       map((data) => {
-        const limitFieldMatch = options.limitField || DEFAULT_LIMIT_FIELD;
+        let limit = DEFAULT_LIMIT_FIELD;
+        if (options.limitField !== undefined) {
+          if (typeof options.limitField === 'string') {
+            if (transformationsVariableSupport()) {
+              limit = parseInt(ctx.interpolate(options.limitField), 10);
+            } else {
+              limit = parseInt(options.limitField, 10);
+            }
+          } else {
+            limit = options.limitField;
+          }
+        }
         return data.map((frame) => {
-          if (frame.length > limitFieldMatch) {
+          if (frame.length > limit) {
             return {
               ...frame,
               fields: frame.fields.map((f) => {
                 return {
                   ...f,
-                  values: f.values.slice(0, limitFieldMatch),
+                  values: f.values.slice(0, limit),
                 };
               }),
-              length: limitFieldMatch,
+              length: limit,
             };
           }
 

packages/grafana-data/src/transformations/transformers/sortBy.ts

@@ -3,9 +3,10 @@ import { map } from 'rxjs/operators';
 import { sortDataFrame } from '../../dataframe';
 import { getFieldDisplayName } from '../../field';
 import { DataFrame } from '../../types';
-import { DataTransformerInfo } from '../../types/transformations';
+import { DataTransformContext, DataTransformerInfo } from '../../types/transformations';
 
 import { DataTransformerID } from './ids';
+import { transformationsVariableSupport } from './utils';
 
 export interface SortByField {
   field: string;
@@ -31,20 +32,20 @@ export const sortByTransformer: DataTransformerInfo<SortByTransformerOptions> =
    * Return a modified copy of the series. If the transform is not or should not
    * be applied, just return the input series
    */
-  operator: (options) => (source) =>
+  operator: (options, ctx) => (source) =>
     source.pipe(
       map((data) => {
         if (!Array.isArray(data) || data.length === 0 || !options?.sort?.length) {
           return data;
         }
-        return sortDataFrames(data, options.sort);
+        return sortDataFrames(data, options.sort, ctx);
       })
     ),
 };
 
-export function sortDataFrames(data: DataFrame[], sort: SortByField[]): DataFrame[] {
+export function sortDataFrames(data: DataFrame[], sort: SortByField[], ctx: DataTransformContext): DataFrame[] {
   return data.map((frame) => {
-    const s = attachFieldIndex(frame, sort);
+    const s = attachFieldIndex(frame, sort, ctx);
     if (s.length && s[0].index != null) {
       return sortDataFrame(frame, s[0].index, s[0].desc);
     }
@@ -52,12 +53,18 @@ export function sortDataFrames(data: DataFrame[], sort: SortByField[]): DataFram
   });
 }
 
-function attachFieldIndex(frame: DataFrame, sort: SortByField[]): SortByField[] {
+function attachFieldIndex(frame: DataFrame, sort: SortByField[], ctx: DataTransformContext): SortByField[] {
   return sort.map((s) => {
     if (s.index != null) {
       // null or undefined
       return s;
     }
+    if (transformationsVariableSupport()) {
+      return {
+        ...s,
+        index: frame.fields.findIndex((f) => ctx.interpolate(s.field) === getFieldDisplayName(f, frame)),
+      };
+    }
     return {
       ...s,
       index: frame.fields.findIndex((f) => s.field === getFieldDisplayName(f, frame)),

packages/grafana-data/src/transformations/transformers/utils.ts

@@ -0,0 +1,3 @@
+export const transformationsVariableSupport = () => {
+  return (window as any)?.grafanaBootData?.settings?.featureToggles?.transformationsVariableSupport;
+};

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -136,4 +136,5 @@ export interface FeatureToggles {
   externalServiceAccounts?: boolean;
   alertingModifiedExport?: boolean;
   enableNativeHTTPHistogram?: boolean;
+  transformationsVariableSupport?: boolean;
 }

packages/grafana-ui/src/components/DataLinks/DataLinkSuggestions.tsx

@@ -31,6 +31,7 @@ const getStyles = (theme: GrafanaTheme2) => {
     item: css({
       background: 'none',
       padding: '2px 8px',
+      userSelect: 'none',
       color: theme.colors.text.primary,
       cursor: 'pointer',
       '&:hover': {