From 388918c2cb2ed14f97304bbf98f1653afac086ee Mon Sep 17 00:00:00 2001
From: Marcus Andersson <marcus.andersson@grafana.com>
Date: Thu, 15 Jul 2021 10:36:18 +0200
Subject: [PATCH] [v7.5.x] Transformations: add 'prepare time series'
 transformer (#36749)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
---
 .../src/transformations/transformers/ids.ts   |   1 +
 .../PrepareTimeSeriesEditor.tsx               |  87 +++++++
 .../prepareTimeSeries.test.ts                 | 214 ++++++++++++++++++
 .../prepareTimeSeries/prepareTimeSeries.ts    |  93 ++++++++
 public/app/core/utils/standardTransformers.ts |   2 +
 5 files changed, 397 insertions(+)
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
 create mode 100644 public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts

diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts
index 1aaf5773610..218e1cb5586 100644
--- a/packages/grafana-data/src/transformations/transformers/ids.ts
+++ b/packages/grafana-data/src/transformations/transformers/ids.ts
@@ -22,4 +22,5 @@ export enum DataTransformerID {
   ensureColumns = 'ensureColumns',
   groupBy = 'groupBy',
   sortBy = 'sortBy',
+  prepareTimeSeries = 'prepareTimeSeries',
 }
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx b/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
new file mode 100644
index 00000000000..90aa1f6fb08
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx
@@ -0,0 +1,87 @@
+import React, { useCallback } from 'react';
+import { css } from 'emotion';
+import { GrafanaTheme, SelectableValue, TransformerRegistryItem, TransformerUIProps } from '@grafana/data';
+import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+import { InlineField, InlineFieldRow, Select, useStyles } from '@grafana/ui';
+
+const wideInfo = {
+  label: 'Wide time series',
+  value: timeSeriesFormat.TimeSeriesWide,
+  description: 'Creates a single frame joined by time',
+  info: (
+    <ul>
+      <li>Single frame</li>
+      <li>1st field is shared time field</li>
+      <li>Time in ascending order</li>
+      <li>Multiple value fields of any type</li>
+    </ul>
+  ),
+};
+
+const manyInfo = {
+  label: 'Multi-frame time series',
+  value: timeSeriesFormat.TimeSeriesMany,
+  description: 'Creates a new frame for each time/number pair',
+  info: (
+    <ul>
+      <li>Multiple frames</li>
+      <li>Each frame has two fields: time, value</li>
+      <li>Time in ascending order</li>
+      <li>All values are numeric</li>
+    </ul>
+  ),
+};
+
+const formats: Array<SelectableValue<timeSeriesFormat>> = [wideInfo, manyInfo];
+
+export function PrepareTimeSeriesEditor(props: TransformerUIProps<PrepareTimeSeriesOptions>): React.ReactElement {
+  const { options, onChange } = props;
+  const styles = useStyles(getStyles);
+
+  const onSelectFormat = useCallback(
+    (value: SelectableValue<timeSeriesFormat>) => {
+      onChange({
+        ...options,
+        format: value.value!,
+      });
+    },
+    [onChange, options]
+  );
+
+  return (
+    <>
+      <InlineFieldRow>
+        <InlineField label="Format" labelWidth={12}>
+          <Select
+            width={35}
+            options={formats}
+            value={formats.find((v) => v.value === options.format) || formats[0]}
+            onChange={onSelectFormat}
+            className="flex-grow-1"
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label="Info" labelWidth={12}>
+          <div className={styles.info}>
+            {options.format === timeSeriesFormat.TimeSeriesMany ? manyInfo.info : wideInfo.info}
+          </div>
+        </InlineField>
+      </InlineFieldRow>
+    </>
+  );
+}
+
+const getStyles = (theme: GrafanaTheme) => ({
+  info: css`
+    margin-left: 20px;
+  `,
+});
+
+export const prepareTimeseriesTransformerRegistryItem: TransformerRegistryItem<PrepareTimeSeriesOptions> = {
+  id: prepareTimeSeriesTransformer.id,
+  editor: PrepareTimeSeriesEditor,
+  transformation: prepareTimeSeriesTransformer,
+  name: prepareTimeSeriesTransformer.name,
+  description: prepareTimeSeriesTransformer.description,
+};
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
new file mode 100644
index 00000000000..cc043766fd4
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
@@ -0,0 +1,214 @@
+import { toDataFrame, ArrayVector, DataFrame, FieldType, toDataFrameDTO, DataFrameDTO } from '@grafana/data';
+import { prepareTimeSeries, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+
+describe('Prepair time series transformer', () => {
+  it('should transform wide to many', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should remove string fields since time series format is expected to be time/number fields', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform all wide to many when mixed', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform none when source only has long frames', () => {
+    const source = [
+      toDataFrame({
+        name: 'long',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(toEquableDataFrames(prepareTimeSeries(source, config))).toEqual(toEquableDataFrames(source));
+  });
+
+  it('should return empty array when no timeseries exist', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+      toDataFrame({
+        name: 'wide',
+        refId: 'B',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([]);
+  });
+});
+
+function toEquableDataFrame(source: any): DataFrame {
+  return toDataFrame({
+    meta: undefined,
+    ...source,
+    fields: source.fields.map((field: any) => {
+      return {
+        ...field,
+        values: new ArrayVector(field.values),
+        config: {},
+      };
+    }),
+  });
+}
+
+function toEquableDataFrames(data: DataFrame[]): DataFrameDTO[] {
+  return data.map((frame) => toDataFrameDTO(frame));
+}
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
new file mode 100644
index 00000000000..197dc77c131
--- /dev/null
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
@@ -0,0 +1,93 @@
+import {
+  DataTransformerInfo,
+  DataFrame,
+  FieldType,
+  DataTransformerID,
+  outerJoinDataFrames,
+  fieldMatchers,
+  FieldMatcherID,
+} from '@grafana/data';
+import { map } from 'rxjs/operators';
+
+/**
+ * There is currently an effort to figure out consistent names
+ * for the various formats/types we produce and use.
+ *
+ * This transformer will eventually include the required metadata that can assert
+ * a DataFrame[] is of a given type
+ *
+ * @internal -- TBD
+ */
+export enum timeSeriesFormat {
+  TimeSeriesWide = 'wide', // [time,...values]
+  TimeSeriesMany = 'many', // All frames have [time,number]
+  //  TimeSeriesLong = 'long',
+}
+
+export type PrepareTimeSeriesOptions = {
+  format: timeSeriesFormat;
+};
+
+/**
+ * Convert to [][time,number]
+ */
+export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] {
+  if (!Array.isArray(data) || data.length === 0) {
+    return data;
+  }
+
+  const result: DataFrame[] = [];
+  for (const frame of data) {
+    const timeField = frame.fields.find((field) => {
+      return field.type === FieldType.time;
+    });
+
+    if (!timeField) {
+      continue;
+    }
+
+    for (const field of frame.fields) {
+      if (field.type !== FieldType.number) {
+        continue;
+      }
+
+      result.push({
+        name: frame.name,
+        refId: frame.refId,
+        meta: frame.meta,
+        fields: [timeField, field],
+        length: frame.length,
+      });
+    }
+  }
+  return result;
+}
+
+export function prepareTimeSeries(data: DataFrame[], options: PrepareTimeSeriesOptions): DataFrame[] {
+  const format = options?.format ?? timeSeriesFormat.TimeSeriesWide;
+  if (format === timeSeriesFormat.TimeSeriesMany) {
+    return toTimeSeriesMany(data);
+  }
+
+  // Join by the first frame
+  const frame = outerJoinDataFrames({
+    frames: data,
+    joinBy: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
+    enforceSort: true,
+    keepOriginIndices: true,
+  });
+  return frame ? [frame] : [];
+}
+
+export const prepareTimeSeriesTransformer: DataTransformerInfo<PrepareTimeSeriesOptions> = {
+  id: DataTransformerID.prepareTimeSeries,
+  name: 'Prepare time series',
+  description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`,
+  defaultOptions: {},
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  operator: (options) => (source) => source.pipe(map((data) => prepareTimeSeries(data, options))),
+};
diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts
index fe5429fde35..390a2c3fd37 100644
--- a/public/app/core/utils/standardTransformers.ts
+++ b/public/app/core/utils/standardTransformers.ts
@@ -13,6 +13,7 @@ import { mergeTransformerRegistryItem } from '../components/TransformersUI/Merge
 import { seriesToRowsTransformerRegistryItem } from '../components/TransformersUI/SeriesToRowsTransformerEditor';
 import { concatenateTransformRegistryItem } from '../components/TransformersUI/ConcatenateTransformerEditor';
 import { renameByRegexTransformRegistryItem } from '../components/TransformersUI/RenameByRegexTransformer';
+import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
   return [
@@ -30,5 +31,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     groupByTransformRegistryItem,
     sortByTransformRegistryItem,
     mergeTransformerRegistryItem,
+    prepareTimeseriesTransformerRegistryItem,
   ];
 };