Transformers: adds labels as columns transformer (#23703)

* Transformers: adds labels as columns transformer

* Refactor: adds support for same timestamps with different labels

* Refactor: adds basic transform ui

* Refactor: adds sorted result

* Refactor: renames transformer

packages/grafana-data/src/transformations/transformers.ts

@@ -9,6 +9,7 @@ import { orderFieldsTransformer } from './transformers/order';
 import { organizeFieldsTransformer } from './transformers/organize';
 import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
 import { renameFieldsTransformer } from './transformers/rename';
+import { labelsToFieldsTransformer } from './transformers/labelsToFields';
 
 export const standardTransformers = {
   noopTransformer,
@@ -23,4 +24,5 @@ export const standardTransformers = {
   calculateFieldTransformer,
   seriesToColumnsTransformer,
   renameFieldsTransformer,
+  labelsToFieldsTransformer,
 };

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -9,6 +9,7 @@ export enum DataTransformerID {
   calculateField = 'calculateField', // Run a reducer on the row
 
   seriesToColumns = 'seriesToColumns', // former table transform timeseries_to_columns
+  labelsToFields = 'labelsToFields', // former table transform table
   filterFields = 'filterFields', // Pick some fields (keep all frames)
   filterFieldsByName = 'filterFieldsByName', // Pick fields with name matching regex (keep all frames)
   filterFrames = 'filterFrames', // Pick some frames (keep all fields)

packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts

@@ -0,0 +1,232 @@
+import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
+import { LabelsToFieldsOptions, labelsToFieldsTransformer } from './labelsToFields';
+import { DataTransformerConfig, Field, FieldType } from '../../types';
+import { DataTransformerID } from './ids';
+import { toDataFrame } from '../../dataframe';
+import { transformDataFrame } from '../transformDataFrame';
+import { ArrayVector } from '../../vector';
+
+describe('Labels as Columns', () => {
+  beforeAll(() => {
+    mockTransformationsRegistry([labelsToFieldsTransformer]);
+  });
+
+  it('data frame with 1 value and 1 label', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const oneValueOneLabelA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000] },
+        { name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside' } },
+      ],
+    });
+
+    const oneValueOneLabelB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [2000] },
+        { name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside' } },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} },
+      { name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+
+  it('data frame with 2 values and 1 label', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const twoValuesOneLabelA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000] },
+        { name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside' } },
+        { name: 'humidity', type: FieldType.number, values: [10000], labels: { location: 'inside' } },
+      ],
+    });
+
+    const twoValuesOneLabelB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [2000] },
+        { name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside' } },
+        { name: 'humidity', type: FieldType.number, values: [11000], labels: { location: 'outside' } },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [twoValuesOneLabelA, twoValuesOneLabelB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} },
+      { name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} },
+      { name: 'humidity', type: FieldType.number, values: new ArrayVector([10000, 11000]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+
+  it('data frame with 1 value and 2 labels', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const oneValueTwoLabelsA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000] },
+        { name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', area: 'living room' } },
+      ],
+    });
+
+    const oneValueTwoLabelsB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [2000] },
+        { name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', area: 'backyard' } },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [oneValueTwoLabelsA, oneValueTwoLabelsB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} },
+      { name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} },
+      { name: 'area', type: FieldType.string, values: new ArrayVector(['living room', 'backyard']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+
+  it('data frame with 2 values and 2 labels', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const twoValuesTwoLabelsA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000] },
+        { name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', area: 'living room' } },
+        {
+          name: 'humidity',
+          type: FieldType.number,
+          values: [10000],
+          labels: { location: 'inside', area: 'living room' },
+        },
+      ],
+    });
+
+    const twoValuesTwoLabelsB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [2000] },
+        { name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', area: 'backyard' } },
+        {
+          name: 'humidity',
+          type: FieldType.number,
+          values: [11000],
+          labels: { location: 'outside', area: 'backyard' },
+        },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [twoValuesTwoLabelsA, twoValuesTwoLabelsB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} },
+      { name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} },
+      { name: 'area', type: FieldType.string, values: new ArrayVector(['living room', 'backyard']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} },
+      { name: 'humidity', type: FieldType.number, values: new ArrayVector([10000, 11000]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+
+  it('data frames with different labels', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const oneValueDifferentLabelsA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000] },
+        { name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', feelsLike: 'ok' } },
+      ],
+    });
+
+    const oneValueDifferentLabelsB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [2000] },
+        { name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', sky: 'cloudy' } },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [oneValueDifferentLabelsA, oneValueDifferentLabelsB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} },
+      { name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} },
+      { name: 'feelsLike', type: FieldType.string, values: new ArrayVector(['ok', null]), config: {} },
+      { name: 'sky', type: FieldType.string, values: new ArrayVector([null, 'cloudy']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+
+  it('data frames with same timestamp and different labels', () => {
+    const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
+      id: DataTransformerID.labelsToFields,
+      options: {},
+    };
+
+    const oneValueDifferentLabelsA = toDataFrame({
+      name: 'A',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000, 2000] },
+        { name: 'temp', type: FieldType.number, values: [1, 2], labels: { location: 'inside', feelsLike: 'ok' } },
+      ],
+    });
+
+    const oneValueDifferentLabelsB = toDataFrame({
+      name: 'B',
+      fields: [
+        { name: 'time', type: FieldType.time, values: [1000, 2000] },
+        { name: 'temp', type: FieldType.number, values: [-1, -2], labels: { location: 'outside', sky: 'cloudy' } },
+      ],
+    });
+
+    const result = transformDataFrame([cfg], [oneValueDifferentLabelsA, oneValueDifferentLabelsB]);
+    const expected: Field[] = [
+      { name: 'time', type: FieldType.time, values: new ArrayVector([1000, 1000, 2000, 2000]), config: {} },
+      {
+        name: 'location',
+        type: FieldType.string,
+        values: new ArrayVector(['inside', 'outside', 'inside', 'outside']),
+        config: {},
+      },
+      { name: 'feelsLike', type: FieldType.string, values: new ArrayVector(['ok', null, 'ok', null]), config: {} },
+      { name: 'sky', type: FieldType.string, values: new ArrayVector([null, 'cloudy', null, 'cloudy']), config: {} },
+      { name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1, 2, -2]), config: {} },
+    ];
+
+    expect(result[0].fields).toEqual(expected);
+  });
+});

packages/grafana-data/src/transformations/transformers/labelsToFields.ts

@@ -0,0 +1,135 @@
+import { DataFrame, DataTransformerInfo, FieldType } from '../../types';
+import { DataTransformerID } from './ids';
+import { MutableDataFrame } from '../../dataframe';
+import { ArrayVector } from '../../vector';
+import { filterFieldsTransformer } from './filter';
+import { FieldMatcherID } from '..';
+
+export interface LabelsToFieldsOptions {}
+type MapItem = { type: FieldType; values: Record<string, any>; isValue: boolean };
+type SeriesMapItem = Record<string, MapItem>;
+type Map = Record<string, SeriesMapItem>;
+
+export const labelsToFieldsTransformer: DataTransformerInfo<LabelsToFieldsOptions> = {
+  id: DataTransformerID.labelsToFields,
+  name: 'Labels to fields',
+  description: 'Groups series by time and return labels as columns',
+  defaultOptions: {},
+  transformer: options => (data: DataFrame[]) => {
+    const framesWithTimeField = filterFieldsTransformer.transformer({ include: { id: FieldMatcherID.time } })(data);
+    if (!framesWithTimeField.length || !framesWithTimeField[0].fields.length) {
+      return data;
+    }
+
+    const framesWithoutTimeField = filterFieldsTransformer.transformer({ exclude: { id: FieldMatcherID.time } })(data);
+    if (!framesWithoutTimeField.length || !framesWithoutTimeField[0].fields.length) {
+      return data;
+    }
+
+    const columnsMap = createColumnsMap(framesWithTimeField, framesWithoutTimeField);
+    const processed = createFields(columnsMap);
+    const values: Record<string, any[]> = {};
+
+    const timeColumnItem = columnsMap[processed.fields[0].name];
+    const seriesIndexStrings = Object.keys(timeColumnItem);
+    for (const seriesIndexString of seriesIndexStrings) {
+      const seriesItem = timeColumnItem[seriesIndexString];
+      const timeValueStrings = Object.keys(seriesItem.values);
+
+      for (const timeValueString of timeValueStrings) {
+        if (!values[timeValueString]) {
+          values[timeValueString] = [];
+        }
+        let row = new Array(processed.fields.length);
+        for (let index = 0; index < processed.fields.length; index++) {
+          const field = processed.fields[index];
+          const valueItem = columnsMap[field.name][seriesIndexString];
+          const value = valueItem ? valueItem.values[timeValueString] ?? null : null;
+          row[index] = value;
+        }
+        values[timeValueString].push(row);
+      }
+    }
+
+    const timestamps = Object.values(values);
+    for (const timestamp of timestamps) {
+      for (const row of timestamp) {
+        for (let fieldIndex = 0; fieldIndex < processed.fields.length; fieldIndex++) {
+          processed.fields[fieldIndex].values.add(row[fieldIndex]);
+        }
+      }
+    }
+
+    return [processed];
+  },
+};
+
+function addOrAppendMapItem(args: { map: Map; series: number; column: string; type: FieldType; isValue?: boolean }) {
+  const { map, column, type, series, isValue = false } = args;
+  // we're using the fact that the series (number) will automatically become a string prop on the object
+  const seriesMapItem: SeriesMapItem = { [series]: { type, values: {}, isValue } };
+  if (!map[column]) {
+    map[column] = seriesMapItem;
+  }
+
+  if (!map[column][series]) {
+    map[column] = { ...map[column], ...seriesMapItem };
+  }
+}
+
+// this is a naive implementation that does the job, not optimized for performance or speed
+function createColumnsMap(framesWithTimeField: DataFrame[], framesWithoutTimeField: DataFrame[]) {
+  const map: Map = {};
+
+  for (let frameIndex = 0; frameIndex < framesWithTimeField.length; frameIndex++) {
+    const timeFrame = framesWithTimeField[frameIndex];
+    const otherFrame = framesWithoutTimeField[frameIndex];
+    const timeField = timeFrame.fields[0];
+    addOrAppendMapItem({ map, column: timeField.name, series: frameIndex, type: timeField.type });
+
+    for (let valueIndex = 0; valueIndex < timeFrame.length; valueIndex++) {
+      const timeFieldValue = timeField.values.get(valueIndex);
+      map[timeField.name][frameIndex].values[timeFieldValue] = timeFieldValue;
+
+      for (const field of otherFrame.fields) {
+        if (field.labels) {
+          const labels = Object.keys(field.labels);
+          for (const label of labels) {
+            addOrAppendMapItem({ map, column: label, series: frameIndex, type: FieldType.string });
+
+            map[label][frameIndex].values[timeFieldValue] = field.labels[label];
+          }
+        }
+
+        const otherFieldValue = field.values.get(valueIndex);
+        addOrAppendMapItem({ map, column: field.name, series: frameIndex, type: field.type, isValue: true });
+
+        map[field.name][frameIndex].values[timeFieldValue] = otherFieldValue;
+      }
+    }
+  }
+
+  return map;
+}
+
+function createFields(columnsMap: Map) {
+  const columns = Object.keys(columnsMap);
+  const processed = new MutableDataFrame();
+  const valueColumns: string[] = [];
+
+  for (const column of columns) {
+    const columnItem = Object.values<MapItem>(columnsMap[column])[0];
+    if (columnItem.isValue) {
+      valueColumns.push(column);
+      continue;
+    }
+    processed.addField({ type: columnItem.type, values: new ArrayVector(), name: column });
+  }
+
+  for (const column of valueColumns) {
+    const columnItem = Object.values<MapItem>(columnsMap[column])[0];
+    processed.addField({ type: columnItem.type, values: new ArrayVector(), name: column });
+  }
+
+  return processed;
+}

packages/grafana-ui/src/components/TransformersUI/LabelsAsFieldsTransformerEditor.tsx

@@ -0,0 +1,19 @@
+import React from 'react';
+import { DataTransformerID, standardTransformers, TransformerRegistyItem, TransformerUIProps } from '@grafana/data';
+import { LabelsToFieldsOptions } from '@grafana/data/src/transformations/transformers/labelsToFields';
+
+export const LabelsAsFieldsTransformerEditor: React.FC<TransformerUIProps<LabelsToFieldsOptions>> = ({
+  input,
+  options,
+  onChange,
+}) => {
+  return null;
+};
+
+export const labelsAsFieldsTransformerRegistryItem: TransformerRegistyItem<LabelsToFieldsOptions> = {
+  id: DataTransformerID.labelsToFields,
+  editor: LabelsAsFieldsTransformerEditor,
+  transformation: standardTransformers.labelsToFieldsTransformer,
+  name: 'Labels as fields',
+  description: 'Groups series by time and return labels as fields',
+};

packages/grafana-ui/src/utils/standardTransformers.ts

@@ -5,6 +5,7 @@ import { filterFramesByRefIdTransformRegistryItem } from '../components/Transfor
 import { organizeFieldsTransformRegistryItem } from '../components/TransformersUI/OrganizeFieldsTransformerEditor';
 import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor';
 import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor';
+import { labelsAsFieldsTransformerRegistryItem } from '../components/TransformersUI/LabelsAsFieldsTransformerEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> => {
   return [
@@ -14,5 +15,6 @@ export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> =>
     organizeFieldsTransformRegistryItem,
     seriesToFieldsTransformerRegistryItem,
     calculateFieldTransformRegistryItem,
+    labelsAsFieldsTransformerRegistryItem,
   ];
 };