From 92801d5fa1d6bfafad653710b45d4f624e17d0f1 Mon Sep 17 00:00:00 2001
From: Ryan McKinley <ryantxu@gmail.com>
Date: Wed, 14 Jul 2021 02:24:12 -0700
Subject: [PATCH] Transformations: add 'prepare time series' transformer
 (#36737)

* adding transformer that will stretch a data frame from wide to long.

* added a UI for the stretch frames transformer.

* refactored according to feedback.

* removed unused dep.

* making sure it is being displayed.

* minor adjustments.

* move stretch to prepare

* improved readability of tests.

* refactored to use a function component syntax.

Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
---
 .../src/transformations/transformers/ids.ts   |   1 +
 .../PrepareTimeSeriesEditor.tsx               |  94 ++++++++
 .../prepareTimeSeries.test.ts                 | 214 ++++++++++++++++++
 .../prepareTimeSeries/prepareTimeSeries.ts    |  93 ++++++++
 public/app/core/utils/standardTransformers.ts |   2 +
 5 files changed, 404 insertions(+)
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts

diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts
index 9018302f605..a09e2075e22 100644
--- a/packages/grafana-data/src/transformations/transformers/ids.ts
+++ b/packages/grafana-data/src/transformations/transformers/ids.ts
@@ -25,4 +25,5 @@ export enum DataTransformerID {
   histogram = 'histogram',
   configFromData = 'configFromData',
   rowsToFields = 'rowsToFields',
+  prepareTimeSeries = 'prepareTimeSeries',
 }
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx b/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
new file mode 100644
index 00000000000..e4f24118804
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
@@ -0,0 +1,94 @@
+import React, { useCallback } from 'react';
+import { GrafanaTheme2, SelectableValue, TransformerRegistryItem, TransformerUIProps } from '@grafana/data';
+import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+import { InlineField, InlineFieldRow, Select, useStyles2 } from '@grafana/ui';
+import { css } from '@emotion/css';
+
+const wideInfo = {
+  label: 'Wide time series',
+  value: timeSeriesFormat.TimeSeriesWide,
+  description: 'Creates a single frame joined by time',
+  info: (
+    <ul>
+      <li>Single frame</li>
+      <li>1st field is shared time field</li>
+      <li>Time in ascending order</li>
+      <li>Multiple value fields of any type</li>
+    </ul>
+  ),
+};
+
+const manyInfo = {
+  label: 'Multi-frame time series',
+  value: timeSeriesFormat.TimeSeriesMany,
+  description: 'Creates a new frame for each time/number pair',
+  info: (
+    <ul>
+      <li>Multiple frames</li>
+      <li>Each frame has two fields: time, value</li>
+      <li>Time in ascending order</li>
+      <li>All values are numeric</li>
+    </ul>
+  ),
+};
+
+const formats: Array<SelectableValue<timeSeriesFormat>> = [wideInfo, manyInfo];
+
+export function PrepareTimeSeriesEditor(props: TransformerUIProps<PrepareTimeSeriesOptions>): React.ReactElement {
+  const { options, onChange } = props;
+  const styles = useStyles2(getStyles);
+
+  const onSelectFormat = useCallback(
+    (value: SelectableValue<timeSeriesFormat>) => {
+      onChange({
+        ...options,
+        format: value.value!,
+      });
+    },
+    [onChange, options]
+  );
+
+  return (
+    <>
+      <InlineFieldRow>
+        <InlineField label="Format" labelWidth={12}>
+          <Select
+            width={35}
+            options={formats}
+            value={formats.find((v) => v.value === options.format) || formats[0]}
+            onChange={onSelectFormat}
+            className="flex-grow-1"
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label="Info" labelWidth={12}>
+          <div className={styles.info}>
+            {options.format === timeSeriesFormat.TimeSeriesMany ? manyInfo.info : wideInfo.info}
+          </div>
+        </InlineField>
+      </InlineFieldRow>
+    </>
+  );
+}
+
+const getStyles = (theme: GrafanaTheme2) => ({
+  info: css`
+    margin-left: 20px;
+  `,
+});
+
+export const prepareTimeseriesTransformerRegistryItem: TransformerRegistryItem<PrepareTimeSeriesOptions> = {
+  id: prepareTimeSeriesTransformer.id,
+  editor: PrepareTimeSeriesEditor,
+  transformation: prepareTimeSeriesTransformer,
+  name: prepareTimeSeriesTransformer.name,
+  description: prepareTimeSeriesTransformer.description,
+  help: `
+  ### Use cases 
+  
+  This will take query results and transform them into a predictable timeseries format.  
+  This transformer may be especially useful when using old panels that only expect the
+  many-frame timeseries format.
+  `,
+};
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
new file mode 100644
index 00000000000..cc043766fd4
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
@@ -0,0 +1,214 @@
+import { toDataFrame, ArrayVector, DataFrame, FieldType, toDataFrameDTO, DataFrameDTO } from '@grafana/data';
+import { prepareTimeSeries, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+
+describe('Prepair time series transformer', () => {
+  it('should transform wide to many', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should remove string fields since time series format is expected to be time/number fields', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform all wide to many when mixed', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform none when source only has long frames', () => {
+    const source = [
+      toDataFrame({
+        name: 'long',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(toEquableDataFrames(prepareTimeSeries(source, config))).toEqual(toEquableDataFrames(source));
+  });
+
+  it('should return empty array when no timeseries exist', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+      toDataFrame({
+        name: 'wide',
+        refId: 'B',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([]);
+  });
+});
+
+function toEquableDataFrame(source: any): DataFrame {
+  return toDataFrame({
+    meta: undefined,
+    ...source,
+    fields: source.fields.map((field: any) => {
+      return {
+        ...field,
+        values: new ArrayVector(field.values),
+        config: {},
+      };
+    }),
+  });
+}
+
+function toEquableDataFrames(data: DataFrame[]): DataFrameDTO[] {
+  return data.map((frame) => toDataFrameDTO(frame));
+}
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
new file mode 100644
index 00000000000..197dc77c131
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
@@ -0,0 +1,93 @@
+import {
+  DataTransformerInfo,
+  DataFrame,
+  FieldType,
+  DataTransformerID,
+  outerJoinDataFrames,
+  fieldMatchers,
+  FieldMatcherID,
+} from '@grafana/data';
+import { map } from 'rxjs/operators';
+
+/**
+ * There is currently an effort to figure out consistent names
+ * for the various formats/types we produce and use.
+ *
+ * This transformer will eventually include the required metadata that can assert
+ * a DataFrame[] is of a given type
+ *
+ * @internal -- TBD
+ */
+export enum timeSeriesFormat {
+  TimeSeriesWide = 'wide', // [time,...values]
+  TimeSeriesMany = 'many', // All frames have [time,number]
+  //  TimeSeriesLong = 'long',
+}
+
+export type PrepareTimeSeriesOptions = {
+  format: timeSeriesFormat;
+};
+
+/**
+ * Convert to [][time,number]
+ */
+export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] {
+  if (!Array.isArray(data) || data.length === 0) {
+    return data;
+  }
+
+  const result: DataFrame[] = [];
+  for (const frame of data) {
+    const timeField = frame.fields.find((field) => {
+      return field.type === FieldType.time;
+    });
+
+    if (!timeField) {
+      continue;
+    }
+
+    for (const field of frame.fields) {
+      if (field.type !== FieldType.number) {
+        continue;
+      }
+
+      result.push({
+        name: frame.name,
+        refId: frame.refId,
+        meta: frame.meta,
+        fields: [timeField, field],
+        length: frame.length,
+      });
+    }
+  }
+  return result;
+}
+
+export function prepareTimeSeries(data: DataFrame[], options: PrepareTimeSeriesOptions): DataFrame[] {
+  const format = options?.format ?? timeSeriesFormat.TimeSeriesWide;
+  if (format === timeSeriesFormat.TimeSeriesMany) {
+    return toTimeSeriesMany(data);
+  }
+
+  // Join by the first frame
+  const frame = outerJoinDataFrames({
+    frames: data,
+    joinBy: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
+    enforceSort: true,
+    keepOriginIndices: true,
+  });
+  return frame ? [frame] : [];
+}
+
+export const prepareTimeSeriesTransformer: DataTransformerInfo<PrepareTimeSeriesOptions> = {
+  id: DataTransformerID.prepareTimeSeries,
+  name: 'Prepare time series',
+  description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`,
+  defaultOptions: {},
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  operator: (options) => (source) => source.pipe(map((data) => prepareTimeSeries(data, options))),
+};
diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts
index 5d436996f8b..16f8b0a67d5 100644
--- a/public/app/core/utils/standardTransformers.ts
+++ b/public/app/core/utils/standardTransformers.ts
@@ -16,6 +16,7 @@ import { renameByRegexTransformRegistryItem } from '../components/TransformersUI
 import { histogramTransformRegistryItem } from '../components/TransformersUI/HistogramTransformerEditor';
 import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/rowsToFields/RowsToFieldsTransformerEditor';
 import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor';
+import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
   return [
@@ -36,5 +37,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     histogramTransformRegistryItem,
     rowsToFieldsTransformRegistryItem,
     configFromQueryTransformRegistryItem,
+    prepareTimeseriesTransformerRegistryItem,
   ];
 };