Dataplane: Support prometheus dataplane contract for transformations and name matchers (#65237)

Co-authored-by: Brendan O'Handley <brendan.ohandley@grafana.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -107,6 +107,7 @@ Alpha features might be changed or removed without prior notice.
 | `alertStateHistoryLokiOnly`        | Disable Grafana alerts from emitting annotations when a remote Loki instance is available.                                                                                                          |
 | `unifiedRequestLog`                | Writes error logs to the request logger                                                                                                                                                             |
 | `pyroscopeFlameGraph`              | Changes flame graph to pyroscope one                                                                                                                                                                |
+| `dataplaneFrontendFallback`        | Support dataplane contract field name change for transformations and field name matchers where the name is different                                                                                |
 
 ## Development feature toggles
 

packages/grafana-data/src/field/fieldState.ts

@@ -55,7 +55,7 @@ export function getFieldDisplayName(field: Field, frame?: DataFrame, allFrames?:
 /**
  * Get an appropriate display name. If the 'displayName' field config is set, use that.
  */
-function calculateFieldDisplayName(field: Field, frame?: DataFrame, allFrames?: DataFrame[]): string {
+export function calculateFieldDisplayName(field: Field, frame?: DataFrame, allFrames?: DataFrame[]): string {
   const hasConfigTitle = field.config?.displayName && field.config?.displayName.length;
 
   let displayName = hasConfigTitle ? field.config!.displayName! : field.name;

packages/grafana-data/src/transformations/matchers/nameMatcher.test.ts

@@ -1,9 +1,21 @@
 import { toDataFrame } from '../../dataframe/processDataFrame';
+import { FieldType, DataFrame } from '../../types';
+import { ArrayVector } from '../../vector';
 import { getFieldMatcher } from '../matchers';
 
 import { FieldMatcherID } from './ids';
 import { ByNamesMatcherMode } from './nameMatcher';
 
+// mock the default window.grafanaBootData settings
+// eslint-disable-next-line
+(window as any).grafanaBootData = {
+  settings: {
+    featureToggles: {
+      dataplaneFrontendFallback: true,
+    },
+  },
+};
+
 describe('Field Name by Regexp Matcher', () => {
   it('Match all with wildcard regex', () => {
     const seriesWithNames = toDataFrame({
@@ -385,6 +397,74 @@ describe('Field Regexp or Names Matcher', () => {
       expect(matcher(field, seriesWithNames, [seriesWithNames])).toBe(true);
     }
   });
+
+  it('Support fallback name matchers', () => {
+    const frame: DataFrame = {
+      fields: [
+        { name: 'time', type: FieldType.time, config: {}, values: new ArrayVector([1, 2]) },
+        {
+          name: 'UP',
+          type: FieldType.number,
+          config: {},
+          values: new ArrayVector([1, 2]),
+          labels: { __name__: 'UP' },
+        },
+      ],
+      name: 'X',
+      length: 2,
+    };
+
+    let matcher = getFieldMatcher({
+      id: FieldMatcherID.byName,
+      options: 'Value',
+    });
+    expect(matcher(frame.fields[0], frame, [])).toBeFalsy();
+    expect(matcher(frame.fields[1], frame, [])).toBeTruthy();
+
+    matcher = getFieldMatcher({
+      id: FieldMatcherID.byName,
+      options: 'Time',
+    });
+    expect(matcher(frame.fields[0], frame, [])).toBeTruthy();
+    expect(matcher(frame.fields[1], frame, [])).toBeFalsy();
+  });
+});
+
+it('Support fallback multiple names matchers', () => {
+  const frame: DataFrame = {
+    fields: [
+      { name: 'time', type: FieldType.time, config: {}, values: new ArrayVector([1, 2]) },
+      {
+        name: 'UP',
+        type: FieldType.number,
+        config: {},
+        values: new ArrayVector([1, 2]),
+        labels: { __name__: 'UP' },
+      },
+    ],
+    name: 'X',
+    length: 2,
+  };
+
+  let matcher = getFieldMatcher({
+    id: FieldMatcherID.byNames,
+    options: {
+      mode: ByNamesMatcherMode.include,
+      names: ['Value'],
+    },
+  });
+  expect(matcher(frame.fields[0], frame, [])).toBeFalsy();
+  expect(matcher(frame.fields[1], frame, [])).toBeTruthy();
+
+  matcher = getFieldMatcher({
+    id: FieldMatcherID.byNames,
+    options: {
+      mode: ByNamesMatcherMode.include,
+      names: ['Time'],
+    },
+  });
+  expect(matcher(frame.fields[0], frame, [])).toBeTruthy();
+  expect(matcher(frame.fields[1], frame, [])).toBeFalsy();
 });
 
 describe('Fields returned by query with refId', () => {

packages/grafana-data/src/transformations/matchers/nameMatcher.ts

@@ -1,10 +1,9 @@
 import { getFieldDisplayName } from '../../field/fieldState';
 import { stringToJsRegex } from '../../text/string';
-import { Field, DataFrame } from '../../types/dataFrame';
+import { Field, DataFrame, FieldType, TIME_SERIES_VALUE_FIELD_NAME } from '../../types/dataFrame';
 import { FieldMatcherInfo, FrameMatcherInfo, FieldMatcher } from '../../types/transformations';
 
 import { FieldMatcherID, FrameMatcherID } from './ids';
-
 export interface RegexpOrNamesMatcherOptions {
   pattern?: string;
   names?: string[];
@@ -40,8 +39,16 @@ const fieldNameMatcher: FieldMatcherInfo<string> = {
   defaultOptions: '',
 
   get: (name: string): FieldMatcher => {
+    const uniqueNames = new Set<string>([name] ?? []);
+
+    const fallback = fieldNameFallback(uniqueNames);
+
     return (field: Field, frame: DataFrame, allFrames: DataFrame[]) => {
-      return name === field.name || getFieldDisplayName(field, frame, allFrames) === name;
+      return (
+        name === field.name ||
+        name === getFieldDisplayName(field, frame, allFrames) ||
+        Boolean(fallback && fallback(field, frame, allFrames))
+      );
     };
   },
 
@@ -63,8 +70,14 @@ const multipleFieldNamesMatcher: FieldMatcherInfo<ByNamesMatcherOptions> = {
     const { names, mode = ByNamesMatcherMode.include } = options;
     const uniqueNames = new Set<string>(names ?? []);
 
+    const fallback = fieldNameFallback(uniqueNames);
+
     const matcher = (field: Field, frame: DataFrame, frames: DataFrame[]) => {
-      return uniqueNames.has(field.name) || uniqueNames.has(getFieldDisplayName(field, frame, frames));
+      return (
+        uniqueNames.has(field.name) ||
+        uniqueNames.has(getFieldDisplayName(field, frame, frames)) ||
+        Boolean(fallback && fallback(field, frame, frames))
+      );
     };
 
     if (mode === ByNamesMatcherMode.exclude) {
@@ -85,6 +98,35 @@ const multipleFieldNamesMatcher: FieldMatcherInfo<ByNamesMatcherOptions> = {
   },
 };
 
+// In an effor to support migrating to a consistent data contract, the
+// naming conventions need to get normalized.  However many existing setups
+// exist that would no longer match names if that changes.  This injects
+// fallback logic when when the data frame has not type version specified
+export function fieldNameFallback(fields: Set<string>) {
+  let fallback: FieldMatcher | undefined = undefined;
+
+  // grafana-data does not have access to runtime so we are accessing the window object
+  // to get access to the feature toggle
+  // eslint-disable-next-line
+  const useMatcherFallback = (window as any)?.grafanaBootData?.settings?.featureToggles?.dataplaneFrontendFallback;
+  if (useMatcherFallback) {
+    if (fields.has(TIME_SERIES_VALUE_FIELD_NAME)) {
+      fallback = (field: Field, frame: DataFrame) => {
+        return (
+          Boolean(field.labels) && // Value was reasonable when the name was set in labels or on the frame
+          field.labels?.__name__ === field.name
+        );
+      };
+    } else if (fields.has('Time') || fields.has('time')) {
+      fallback = (field: Field, frame: DataFrame) => {
+        return frame.meta?.typeVersion == null && field.type === FieldType.time;
+      };
+    }
+  }
+
+  return fallback;
+}
+
 const regexpFieldNameMatcher: FieldMatcherInfo<string> = {
   id: FieldMatcherID.byRegexp,
   name: 'Field Name by Regexp',

packages/grafana-data/src/transformations/transformers/groupingToMatrix.ts

@@ -3,6 +3,8 @@ import { map } from 'rxjs/operators';
 import { MutableDataFrame } from '../../dataframe';
 import { getFieldDisplayName } from '../../field/fieldState';
 import { DataFrame, DataTransformerInfo, Field, FieldType, SpecialValue, Vector } from '../../types';
+import { fieldMatchers } from '../matchers';
+import { FieldMatcherID } from '../matchers/ids';
 
 import { DataTransformerID } from './ids';
 
@@ -18,6 +20,11 @@ const DEFAULT_ROW_FIELD = 'Time';
 const DEFAULT_VALUE_FIELD = 'Value';
 const DEFAULT_EMPTY_VALUE = SpecialValue.Empty;
 
+// grafana-data does not have access to runtime so we are accessing the window object
+// to get access to the feature toggle
+// eslint-disable-next-line
+const supportDataplaneFallback = (window as any)?.grafanaBootData?.settings?.featureToggles?.dataplaneFrontendFallback;
+
 export const groupingToMatrixTransformer: DataTransformerInfo<GroupingToMatrixTransformerOptions> = {
   id: DataTransformerID.groupingToMatrix,
   name: 'Grouping to Matrix',
@@ -83,6 +90,14 @@ export const groupingToMatrixTransformer: DataTransformerInfo<GroupingToMatrixTr
             values.push(value);
           }
 
+          // setting the displayNameFromDS in prometheus overrides
+          // the column name based on value fields that are numbers
+          // this prevents columns that should be named 1000190
+          // from becoming named {__name__: 'metricName'}
+          if (supportDataplaneFallback && typeof columnName === 'number') {
+            valueField.config = { ...valueField.config, displayNameFromDS: undefined };
+          }
+
           resultFrame.addField({
             name: columnName.toString(),
             values: values,
@@ -110,7 +125,16 @@ function findKeyField(frame: DataFrame, matchTitle: string): Field | null {
   for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
     const field = frame.fields[fieldIndex];
 
-    if (matchTitle === getFieldDisplayName(field)) {
+    // support for dataplane contract with Prometheus and change in location of field name
+    let matches: boolean;
+    if (supportDataplaneFallback) {
+      const matcher = fieldMatchers.get(FieldMatcherID.byName).get(matchTitle);
+      matches = matcher(field, frame, [frame]);
+    } else {
+      matches = matchTitle === getFieldDisplayName(field);
+    }
+
+    if (matches) {
       return field;
     }
   }

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -94,4 +94,5 @@ export interface FeatureToggles {
   unifiedRequestLog?: boolean;
   renderAuthJWT?: boolean;
   pyroscopeFlameGraph?: boolean;
+  dataplaneFrontendFallback?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -505,5 +505,12 @@ var (
 			State:       FeatureStateAlpha,
 			Owner:       grafanaObservabilityTracesAndProfilingSquad,
 		},
+		{
+			Name:         "dataplaneFrontendFallback",
+			Description:  "Support dataplane contract field name change for transformations and field name matchers where the name is different",
+			State:        FeatureStateAlpha,
+			FrontendOnly: true,
+			Owner:        grafanaObservabilityMetricsSquad,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -75,3 +75,4 @@ alertStateHistoryLokiOnly,alpha,@grafana/alerting-squad,false,false,false,false
 unifiedRequestLog,alpha,@grafana/backend-platform,false,false,false,false
 renderAuthJWT,beta,@grafana/grafana-as-code,false,false,false,false
 pyroscopeFlameGraph,alpha,@grafana/observability-traces-and-profiling,false,false,false,false
+dataplaneFrontendFallback,alpha,@grafana/observability-metrics,false,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -310,4 +310,8 @@ const (
 	// FlagPyroscopeFlameGraph
 	// Changes flame graph to pyroscope one
 	FlagPyroscopeFlameGraph = "pyroscopeFlameGraph"
+
+	// FlagDataplaneFrontendFallback
+	// Support dataplane contract field name change for transformations and field name matchers where the name is different
+	FlagDataplaneFrontendFallback = "dataplaneFrontendFallback"
 )

public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx

@@ -27,7 +27,7 @@ export const TransformationFilter = ({ index, data, config, onChange }: Transfor
 
   return (
     <div className={styles.wrapper}>
-      <Field label="Apply tranformation to">
+      <Field label="Apply transformation to">
         <FrameSelectionEditor
           value={config.filter!}
           context={context}

public/app/plugins/datasource/prometheus/result_transformer.test.ts

@@ -22,6 +22,11 @@ jest.mock('@grafana/runtime', () => ({
       },
     };
   },
+  config: {
+    featureToggles: {
+      prometheusDataplane: true,
+    },
+  },
 }));
 
 const matrixResponse = {
@@ -106,6 +111,74 @@ describe('Prometheus Result Transformer', () => {
       });
     });
 
+    it('dataplane handling, adds displayNameFromDs from calculateFieldDisplayName() when __name__ is the field name when legendFormat is auto', () => {
+      const request = {
+        targets: [
+          {
+            format: 'time_series',
+            refId: 'A',
+            legendFormat: '__auto',
+          },
+        ],
+      } as unknown as DataQueryRequest<PromQuery>;
+      const response = {
+        state: 'Done',
+        data: [
+          {
+            fields: [
+              {
+                name: 'Time',
+                type: 'time',
+                values: [1],
+                typeInfo: { frame: 'time.Time' },
+              },
+              {
+                name: 'up',
+                labels: { __name__: 'up' },
+                config: {},
+                values: [1],
+              },
+            ],
+            length: 1,
+            refId: 'A',
+            meta: {
+              type: 'timeseries-multi',
+              typeVersion: [0, 1],
+            },
+          },
+        ],
+      } as unknown as DataQueryResponse;
+      const series = transformV2(response, request, {});
+      expect(series).toEqual({
+        data: [
+          {
+            fields: [
+              {
+                name: 'Time',
+                type: 'time',
+                values: [1],
+                typeInfo: { frame: 'time.Time' },
+              },
+              {
+                config: { displayNameFromDS: 'up' },
+                labels: { __name__: 'up' },
+                name: 'up',
+                values: [1],
+              },
+            ],
+            length: 1,
+            meta: {
+              type: 'timeseries-multi',
+              typeVersion: [0, 1],
+              preferredVisualisationType: 'graph',
+            },
+            refId: 'A',
+          },
+        ],
+        state: 'Done',
+      });
+    });
+
     it('results with table format should be transformed to table dataFrames', () => {
       const request = {
         targets: [

public/app/plugins/datasource/prometheus/result_transformer.ts

@@ -22,7 +22,8 @@ import {
   TIME_SERIES_TIME_FIELD_NAME,
   TIME_SERIES_VALUE_FIELD_NAME,
 } from '@grafana/data';
-import { FetchResponse, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime';
+import { calculateFieldDisplayName } from '@grafana/data/src/field/fieldState';
+import { config, FetchResponse, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime';
 
 import { renderLegendFormat } from './legend';
 import {
@@ -71,6 +72,23 @@ export function transformV2(
   request: DataQueryRequest<PromQuery>,
   options: { exemplarTraceIdDestinations?: ExemplarTraceIdDestination[] }
 ) {
+  // migration for dataplane field name issue
+  if (config.featureToggles.prometheusDataplane) {
+    // update displayNameFromDS in the field config
+    response.data.forEach((f: DataFrame) => {
+      const target = request.targets.find((t) => t.refId === f.refId);
+      // check that the legend is selected as auto
+      if (target && target.legendFormat === '__auto') {
+        f.fields.forEach((field) => {
+          if (field.labels?.__name__ && field.labels?.__name__ === field.name) {
+            const fieldCopy = { ...field, name: TIME_SERIES_VALUE_FIELD_NAME };
+            field.config.displayNameFromDS = calculateFieldDisplayName(fieldCopy, f, response.data);
+          }
+        });
+      }
+    });
+  }
+
   const [tableFrames, framesWithoutTable] = partition<DataFrame>(response.data, (df) => isTableResult(df, request));
   const processedTableFrames = transformDFToTable(tableFrames);