Transform: improve the "outer join" performance/behavior (#30407)

packages/grafana-data/src/transformations/index.ts

@@ -11,4 +11,4 @@ export {
 } from './standardTransformersRegistry';
 export { RegexpOrNamesMatcherOptions, ByNamesMatcherOptions, ByNamesMatcherMode } from './matchers/nameMatcher';
 export { RenameByRegexTransformerOptions } from './transformers/renameByRegex';
-export { outerJoinDataFrames } from './transformers/seriesToColumns';
+export { outerJoinDataFrames } from './transformers/joinDataFrames';

packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts

@@ -49,52 +49,82 @@ describe('ensureColumns transformer', () => {
 
       const frame = filtered[0];
       expect(frame.fields.length).toEqual(5);
-      expect(filtered[0]).toEqual(
+      expect(filtered[0]).toMatchInlineSnapshot(`
-        toDataFrame({
+        Object {
-          fields: [
+          "fields": Array [
-            {
+            Object {
-              name: 'TheTime',
+              "config": Object {},
-              type: 'time',
+              "name": "TheTime",
-              config: {},
+              "state": Object {
-              values: [1000, 2000],
+                "displayName": "TheTime",
-              labels: undefined,
+              },
+              "type": "time",
+              "values": Array [
+                1000,
+                2000,
+              ],
             },
-            {
+            Object {
-              name: 'A',
+              "config": Object {},
-              type: 'number',
+              "labels": Object {},
-              config: {},
+              "name": "A",
-              values: [1, 100],
+              "state": Object {
-              labels: {},
+                "displayName": "A",
+              },
+              "type": "number",
+              "values": Array [
+                1,
+                100,
+              ],
             },
-            {
+            Object {
-              name: 'B',
+              "config": Object {},
-              type: 'number',
+              "labels": Object {},
-              config: {},
+              "name": "B",
-              values: [2, 200],
+              "state": Object {
-              labels: {},
+                "displayName": "B",
+              },
+              "type": "number",
+              "values": Array [
+                2,
+                200,
+              ],
             },
-            {
+            Object {
-              name: 'C',
+              "config": Object {},
-              type: 'number',
+              "labels": Object {},
-              config: {},
+              "name": "C",
-              values: [3, 300],
+              "state": Object {
-              labels: {},
+                "displayName": "C",
+              },
+              "type": "number",
+              "values": Array [
+                3,
+                300,
+              ],
             },
-            {
+            Object {
-              name: 'D',
+              "config": Object {},
-              type: 'string',
+              "labels": Object {},
-              config: {},
+              "name": "D",
-              values: ['first', 'second'],
+              "state": Object {
-              labels: {},
+                "displayName": "D",
+              },
+              "type": "string",
+              "values": Array [
+                "first",
+                "second",
+              ],
             },
           ],
-          meta: {
+          "length": 2,
-            transformations: ['ensureColumns'],
+          "meta": Object {
+            "transformations": Array [
+              "ensureColumns",
+            ],
           },
-          name: undefined,
+        }
-          refId: undefined,
+      `);
-        })
-      );
     });
   });
 

packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts

@@ -0,0 +1,302 @@
+import { toDataFrame } from '../../dataframe/processDataFrame';
+import { FieldType } from '../../types/dataFrame';
+import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
+import { ArrayVector } from '../../vector';
+import { calculateFieldTransformer } from './calculateField';
+import { isLikelyAscendingVector, outerJoinDataFrames } from './joinDataFrames';
+
+describe('align frames', () => {
+  beforeAll(() => {
+    mockTransformationsRegistry([calculateFieldTransformer]);
+  });
+
+  it('by first time field', () => {
+    const series1 = toDataFrame({
+      fields: [
+        { name: 'TheTime', type: FieldType.time, values: [1000, 2000] },
+        { name: 'A', type: FieldType.number, values: [1, 100] },
+      ],
+    });
+
+    const series2 = toDataFrame({
+      fields: [
+        { name: '_time', type: FieldType.time, values: [1000, 1500, 2000] },
+        { name: 'A', type: FieldType.number, values: [2, 20, 200] },
+        { name: 'B', type: FieldType.number, values: [3, 30, 300] },
+        { name: 'C', type: FieldType.string, values: ['first', 'second', 'third'] },
+      ],
+    });
+
+    const out = outerJoinDataFrames({ frames: [series1, series2] })!;
+    expect(
+      out.fields.map((f) => ({
+        name: f.name,
+        values: f.values.toArray(),
+      }))
+    ).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": "TheTime",
+          "values": Array [
+            1000,
+            1500,
+            2000,
+          ],
+        },
+        Object {
+          "name": "A",
+          "values": Array [
+            1,
+            undefined,
+            100,
+          ],
+        },
+        Object {
+          "name": "A",
+          "values": Array [
+            2,
+            20,
+            200,
+          ],
+        },
+        Object {
+          "name": "B",
+          "values": Array [
+            3,
+            30,
+            300,
+          ],
+        },
+        Object {
+          "name": "C",
+          "values": Array [
+            "first",
+            "second",
+            "third",
+          ],
+        },
+      ]
+    `);
+  });
+
+  it('unsorted input keep indexes', () => {
+    //----------
+    const series1 = toDataFrame({
+      fields: [
+        { name: 'TheTime', type: FieldType.time, values: [1000, 2000, 1500] },
+        { name: 'A1', type: FieldType.number, values: [1, 2, 15] },
+      ],
+    });
+
+    const series3 = toDataFrame({
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [2000, 1000] },
+        { name: 'A2', type: FieldType.number, values: [2, 1] },
+      ],
+    });
+
+    let out = outerJoinDataFrames({ frames: [series1, series3], keepOriginIndices: true })!;
+    expect(
+      out.fields.map((f) => ({
+        name: f.name,
+        values: f.values.toArray(),
+        state: f.state,
+      }))
+    ).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": "TheTime",
+          "state": Object {
+            "displayName": "TheTime",
+            "origin": Object {
+              "fieldIndex": 0,
+              "frameIndex": 0,
+            },
+          },
+          "values": Array [
+            1000,
+            1500,
+            2000,
+          ],
+        },
+        Object {
+          "name": "A1",
+          "state": Object {
+            "displayName": "A1",
+            "origin": Object {
+              "fieldIndex": 1,
+              "frameIndex": 0,
+            },
+          },
+          "values": Array [
+            1,
+            15,
+            2,
+          ],
+        },
+        Object {
+          "name": "A2",
+          "state": Object {
+            "displayName": "A2",
+            "origin": Object {
+              "fieldIndex": 1,
+              "frameIndex": 1,
+            },
+          },
+          "values": Array [
+            1,
+            undefined,
+            2,
+          ],
+        },
+      ]
+    `);
+
+    // Fast path still adds origin indecies
+    out = outerJoinDataFrames({ frames: [series1], keepOriginIndices: true })!;
+    expect(
+      out.fields.map((f) => ({
+        name: f.name,
+        state: f.state,
+      }))
+    ).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": "TheTime",
+          "state": Object {
+            "displayName": "TheTime",
+            "origin": Object {
+              "fieldIndex": 0,
+              "frameIndex": 0,
+            },
+          },
+        },
+        Object {
+          "name": "A1",
+          "state": Object {
+            "displayName": "A1",
+            "origin": Object {
+              "fieldIndex": 1,
+              "frameIndex": 0,
+            },
+          },
+        },
+      ]
+    `);
+  });
+
+  it('sort single frame', () => {
+    const series1 = toDataFrame({
+      fields: [
+        { name: 'TheTime', type: FieldType.time, values: [6000, 2000, 1500] },
+        { name: 'A1', type: FieldType.number, values: [1, 22, 15] },
+      ],
+    });
+
+    const out = outerJoinDataFrames({ frames: [series1], enforceSort: true, keepOriginIndices: true })!;
+    expect(
+      out.fields.map((f) => ({
+        name: f.name,
+        values: f.values.toArray(),
+      }))
+    ).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": "TheTime",
+          "values": Array [
+            1500,
+            2000,
+            6000,
+          ],
+        },
+        Object {
+          "name": "A1",
+          "values": Array [
+            15,
+            22,
+            1,
+          ],
+        },
+      ]
+    `);
+  });
+
+  it('supports duplicate times', () => {
+    //----------
+    // NOTE!!!
+    // * ideally we would *keep* dupicate fields
+    //----------
+    const series1 = toDataFrame({
+      fields: [
+        { name: 'TheTime', type: FieldType.time, values: [1000, 2000] },
+        { name: 'A', type: FieldType.number, values: [1, 100] },
+      ],
+    });
+
+    const series3 = toDataFrame({
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [1000, 1000, 1000] },
+        { name: 'A', type: FieldType.number, values: [2, 20, 200] },
+      ],
+    });
+
+    const out = outerJoinDataFrames({ frames: [series1, series3] })!;
+    expect(
+      out.fields.map((f) => ({
+        name: f.name,
+        values: f.values.toArray(),
+      }))
+    ).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": "TheTime",
+          "values": Array [
+            1000,
+            2000,
+          ],
+        },
+        Object {
+          "name": "A",
+          "values": Array [
+            1,
+            100,
+          ],
+        },
+        Object {
+          "name": "A",
+          "values": Array [
+            200,
+            undefined,
+          ],
+        },
+      ]
+    `);
+  });
+
+  describe('check ascending data', () => {
+    it('simple ascending', () => {
+      const v = new ArrayVector([1, 2, 3, 4, 5]);
+      expect(isLikelyAscendingVector(v)).toBeTruthy();
+    });
+    it('simple ascending with null', () => {
+      const v = new ArrayVector([null, 2, 3, 4, null]);
+      expect(isLikelyAscendingVector(v)).toBeTruthy();
+    });
+    it('single value', () => {
+      const v = new ArrayVector([null, null, null, 4, null]);
+      expect(isLikelyAscendingVector(v)).toBeTruthy();
+      expect(isLikelyAscendingVector(new ArrayVector([4]))).toBeTruthy();
+      expect(isLikelyAscendingVector(new ArrayVector([]))).toBeTruthy();
+    });
+
+    it('middle values', () => {
+      const v = new ArrayVector([null, null, 5, 4, null]);
+      expect(isLikelyAscendingVector(v)).toBeFalsy();
+    });
+
+    it('decending', () => {
+      expect(isLikelyAscendingVector(new ArrayVector([7, 6, null]))).toBeFalsy();
+      expect(isLikelyAscendingVector(new ArrayVector([7, 8, 6]))).toBeFalsy();
+    });
+  });
+});

packages/grafana-data/src/transformations/transformers/joinDataFrames.ts

@@ -0,0 +1,313 @@
+import { DataFrame, Field, FieldMatcher, FieldType, Vector } from '../../types';
+import { ArrayVector } from '../../vector';
+import { fieldMatchers } from '../matchers';
+import { FieldMatcherID } from '../matchers/ids';
+import { getTimeField, sortDataFrame } from '../../dataframe';
+import { getFieldDisplayName } from '../../field';
+
+export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
+  const { timeField } = getTimeField(data[0]);
+  if (timeField) {
+    return fieldMatchers.get(FieldMatcherID.firstTimeField).get({});
+  }
+  let common: string[] = [];
+  for (const f of data[0].fields) {
+    if (f.type === FieldType.number) {
+      common.push(f.name);
+    }
+  }
+
+  for (let i = 1; i < data.length; i++) {
+    const names: string[] = [];
+    for (const f of data[0].fields) {
+      if (f.type === FieldType.number) {
+        names.push(f.name);
+      }
+    }
+    common = common.filter((v) => !names.includes(v));
+  }
+
+  return fieldMatchers.get(FieldMatcherID.byName).get(common[0]);
+}
+
+/**
+ * @alpha
+ */
+export interface JoinOptions {
+  /**
+   * The input fields
+   */
+  frames: DataFrame[];
+
+  /**
+   * The field to join -- frames that do not have this field will be droppped
+   */
+  joinBy?: FieldMatcher;
+
+  /**
+   * Optionally filter the non-join fields
+   */
+  keep?: FieldMatcher;
+
+  /**
+   * When the result is a single frame, this will to a quick check to see if the values are sorted,
+   * and sort if necessary.  If the first/last values are in order the whole vector is assumed to be
+   * sorted
+   */
+  enforceSort?: boolean;
+
+  /**
+   * @internal -- used when we need to keep a reference to the original frame/field index
+   */
+  keepOriginIndices?: boolean;
+}
+
+function getJoinMatcher(options: JoinOptions): FieldMatcher {
+  return options.joinBy ?? pickBestJoinField(options.frames);
+}
+
+/**
+ * This will return a single frame joined by the first matching field.  When a join field is not specified,
+ * the default will use the first time field
+ */
+export function outerJoinDataFrames(options: JoinOptions): DataFrame | undefined {
+  if (!options.frames?.length) {
+    return undefined;
+  }
+
+  if (options.frames.length === 1) {
+    let frame = options.frames[0];
+    if (options.keepOriginIndices) {
+      frame = {
+        ...frame,
+        fields: frame.fields.map((f, fieldIndex) => {
+          const copy = { ...f };
+          const origin = {
+            frameIndex: 0,
+            fieldIndex,
+          };
+          if (copy.state) {
+            copy.state.origin = origin;
+          } else {
+            copy.state = { origin };
+          }
+          return copy;
+        }),
+      };
+    }
+    if (options.enforceSort) {
+      const joinFieldMatcher = getJoinMatcher(options);
+      const joinIndex = frame.fields.findIndex((f) => joinFieldMatcher(f, frame, options.frames));
+      if (joinIndex >= 0) {
+        if (!isLikelyAscendingVector(frame.fields[joinIndex].values)) {
+          return sortDataFrame(frame, joinIndex);
+        }
+      }
+    }
+    return frame;
+  }
+
+  const nullModes: JoinNullMode[][] = [];
+  const allData: AlignedData[] = [];
+  const originalFields: Field[] = [];
+  const joinFieldMatcher = getJoinMatcher(options);
+
+  for (let frameIndex = 0; frameIndex < options.frames.length; frameIndex++) {
+    const frame = options.frames[frameIndex];
+    if (!frame || !frame.fields?.length) {
+      continue; // skip the frame
+    }
+    const nullModesFrame: JoinNullMode[] = [NULL_REMOVE];
+
+    let join: Field | undefined = undefined;
+    let fields: Field[] = [];
+    for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
+      const field = frame.fields[fieldIndex];
+      getFieldDisplayName(field, frame, options.frames); // cache displayName in state
+
+      if (!join && joinFieldMatcher(field, frame, options.frames)) {
+        join = field;
+      } else {
+        if (options.keep && !options.keep(field, frame, options.frames)) {
+          continue; // skip field
+        }
+
+        // Support the standard graph span nulls field config
+        nullModesFrame.push(field.config.custom?.spanNulls ? NULL_REMOVE : NULL_EXPAND);
+
+        let labels = field.labels ?? {};
+        if (frame.name) {
+          labels = { ...labels, name: frame.name };
+        }
+
+        fields.push({
+          ...field,
+          labels, // add the name label from frame
+        });
+      }
+
+      if (options.keepOriginIndices) {
+        field.state!.origin = {
+          frameIndex,
+          fieldIndex,
+        };
+      }
+    }
+
+    if (!join) {
+      continue; // skip the frame
+    }
+
+    if (originalFields.length === 0) {
+      originalFields.push(join); // first join field
+    }
+    nullModes.push(nullModesFrame);
+
+    const a: AlignedData = [join.values.toArray()]; //
+    for (const field of fields) {
+      a.push(field.values.toArray());
+      originalFields.push(field);
+    }
+    allData.push(a);
+  }
+
+  const joined = join(allData, nullModes);
+  return {
+    // ...options.data[0], // keep name, meta?
+    length: joined[0].length,
+    fields: originalFields.map((f, index) => ({
+      ...f,
+      values: new ArrayVector(joined[index]),
+    })),
+  };
+}
+
+//--------------------------------------------------------------------------------
+// Below here is copied from uplot (MIT License)
+// https://github.com/leeoniya/uPlot/blob/master/src/utils.js#L325
+// This avoids needing to import uplot into the data package
+//--------------------------------------------------------------------------------
+
+// Copied from uplot
+type AlignedData = [number[], ...Array<Array<number | null>>];
+
+// nullModes
+const NULL_REMOVE = 0; // nulls are converted to undefined (e.g. for spanGaps: true)
+const NULL_RETAIN = 1; // nulls are retained, with alignment artifacts set to undefined (default)
+const NULL_EXPAND = 2; // nulls are expanded to include any adjacent alignment artifacts
+
+type JoinNullMode = number; // NULL_IGNORE | NULL_RETAIN | NULL_EXPAND;
+
+// sets undefined values to nulls when adjacent to existing nulls (minesweeper)
+function nullExpand(yVals: Array<number | null>, nullIdxs: number[], alignedLen: number) {
+  for (let i = 0, xi, lastNullIdx = -1; i < nullIdxs.length; i++) {
+    let nullIdx = nullIdxs[i];
+
+    if (nullIdx > lastNullIdx) {
+      xi = nullIdx - 1;
+      while (xi >= 0 && yVals[xi] == null) {
+        yVals[xi--] = null;
+      }
+
+      xi = nullIdx + 1;
+      while (xi < alignedLen && yVals[xi] == null) {
+        yVals[(lastNullIdx = xi++)] = null;
+      }
+    }
+  }
+}
+
+// nullModes is a tables-matched array indicating how to treat nulls in each series
+function join(tables: AlignedData[], nullModes: number[][]) {
+  const xVals = new Set<number>();
+
+  for (let ti = 0; ti < tables.length; ti++) {
+    let t = tables[ti];
+    let xs = t[0];
+    let len = xs.length;
+
+    for (let i = 0; i < len; i++) {
+      xVals.add(xs[i]);
+    }
+  }
+
+  let data = [Array.from(xVals).sort((a, b) => a - b)];
+
+  let alignedLen = data[0].length;
+
+  let xIdxs = new Map();
+
+  for (let i = 0; i < alignedLen; i++) {
+    xIdxs.set(data[0][i], i);
+  }
+
+  for (let ti = 0; ti < tables.length; ti++) {
+    let t = tables[ti];
+    let xs = t[0];
+
+    for (let si = 1; si < t.length; si++) {
+      let ys = t[si];
+
+      let yVals = Array(alignedLen).fill(undefined);
+
+      let nullMode = nullModes ? nullModes[ti][si] : NULL_RETAIN;
+
+      let nullIdxs = [];
+
+      for (let i = 0; i < ys.length; i++) {
+        let yVal = ys[i];
+        let alignedIdx = xIdxs.get(xs[i]);
+
+        if (yVal == null) {
+          if (nullMode !== NULL_REMOVE) {
+            yVals[alignedIdx] = yVal;
+
+            if (nullMode === NULL_EXPAND) {
+              nullIdxs.push(alignedIdx);
+            }
+          }
+        } else {
+          yVals[alignedIdx] = yVal;
+        }
+      }
+
+      nullExpand(yVals, nullIdxs, alignedLen);
+
+      data.push(yVals);
+    }
+  }
+
+  return data;
+}
+
+// Quick test if the first and last points look to be ascending
+// Only exported for tests
+export function isLikelyAscendingVector(data: Vector): boolean {
+  let first: any = undefined;
+
+  for (let idx = 0; idx < data.length; idx++) {
+    const v = data.get(idx);
+    if (v != null) {
+      if (first != null) {
+        if (first > v) {
+          return false; // descending
+        }
+        break;
+      }
+      first = v;
+    }
+  }
+
+  let idx = data.length - 1;
+  while (idx >= 0) {
+    const v = data.get(idx--);
+    if (v != null) {
+      if (first > v) {
+        return false;
+      }
+      return true;
+    }
+  }
+
+  return true; // only one non-null point
+}

packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts

@@ -2,7 +2,6 @@ import {
   ArrayVector,
   DataTransformerConfig,
   DataTransformerID,
-  Field,
   FieldType,
   toDataFrame,
   transformDataFrame,
@@ -45,58 +44,102 @@ describe('SeriesToColumns Transformer', () => {
       (received) => {
         const data = received[0];
         const filtered = data[0];
-        expect(filtered.fields).toEqual([
+        expect(filtered.fields).toMatchInlineSnapshot(`
-          {
+          Array [
-            name: 'time',
+            Object {
-            state: {
+              "config": Object {},
-              displayName: 'time',
+              "name": "time",
+              "state": Object {
+                "displayName": "time",
+              },
+              "type": "time",
+              "values": Array [
+                1000,
+                3000,
+                4000,
+                5000,
+                6000,
+                7000,
+              ],
             },
-            type: FieldType.time,
+            Object {
-            values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: undefined,
+                "name": "even",
-          },
+              },
-          {
+              "name": "temperature",
-            name: 'temperature',
+              "state": Object {
-            state: {
+                "displayName": "even temperature",
-              displayName: 'temperature even',
+              },
+              "type": "number",
+              "values": Array [
+                undefined,
+                10.3,
+                10.4,
+                10.5,
+                10.6,
+                undefined,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'even' },
+                "name": "even",
-          },
+              },
-          {
+              "name": "humidity",
-            name: 'humidity',
+              "state": Object {
-            state: {
+                "displayName": "even humidity",
-              displayName: 'humidity even',
+              },
+              "type": "number",
+              "values": Array [
+                undefined,
+                10000.3,
+                10000.4,
+                10000.5,
+                10000.6,
+                undefined,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'even' },
+                "name": "odd",
-          },
+              },
-          {
+              "name": "temperature",
-            name: 'temperature',
+              "state": Object {
-            state: {
+                "displayName": "odd temperature",
-              displayName: 'temperature odd',
+              },
+              "type": "number",
+              "values": Array [
+                11.1,
+                11.3,
+                undefined,
+                11.5,
+                undefined,
+                11.7,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'odd' },
+                "name": "odd",
-          },
+              },
-          {
+              "name": "humidity",
-            name: 'humidity',
+              "state": Object {
-            state: {
+                "displayName": "odd humidity",
-              displayName: 'humidity odd',
+              },
+              "type": "number",
+              "values": Array [
+                11000.1,
+                11000.3,
+                undefined,
+                11000.5,
+                undefined,
+                11000.7,
+              ],
             },
-            type: FieldType.number,
+          ]
-            values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
+        `);
-            config: {},
-            labels: { name: 'odd' },
-          },
-        ]);
       }
     );
   });
@@ -113,58 +156,7 @@ describe('SeriesToColumns Transformer', () => {
       (received) => {
         const data = received[0];
         const filtered = data[0];
-        expect(filtered.fields).toEqual([
+        expect(filtered.fields).toMatchInlineSnapshot(`Array []`);
-          {
-            name: 'temperature',
-            state: {
-              displayName: 'temperature',
-            },
-            type: FieldType.number,
-            values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]),
-            config: {},
-            labels: undefined,
-          },
-          {
-            name: 'time',
-            state: {
-              displayName: 'time even',
-            },
-            type: FieldType.time,
-            values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]),
-            config: {},
-            labels: { name: 'even' },
-          },
-          {
-            name: 'humidity',
-            state: {
-              displayName: 'humidity even',
-            },
-            type: FieldType.number,
-            values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]),
-            config: {},
-            labels: { name: 'even' },
-          },
-          {
-            name: 'time',
-            state: {
-              displayName: 'time odd',
-            },
-            type: FieldType.time,
-            values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]),
-            config: {},
-            labels: { name: 'odd' },
-          },
-          {
-            name: 'humidity',
-            state: {
-              displayName: 'humidity odd',
-            },
-            type: FieldType.number,
-            values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]),
-            config: {},
-            labels: { name: 'odd' },
-          },
-        ]);
       }
     );
   });
@@ -185,58 +177,102 @@ describe('SeriesToColumns Transformer', () => {
       (received) => {
         const data = received[0];
         const filtered = data[0];
-        expect(filtered.fields).toEqual([
+        expect(filtered.fields).toMatchInlineSnapshot(`
-          {
+          Array [
-            name: 'time',
+            Object {
-            state: {
+              "config": Object {},
-              displayName: 'time',
+              "name": "time",
+              "state": Object {
+                "displayName": "time",
+              },
+              "type": "time",
+              "values": Array [
+                1000,
+                3000,
+                4000,
+                5000,
+                6000,
+                7000,
+              ],
             },
-            type: FieldType.time,
+            Object {
-            values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: undefined,
+                "name": "even",
-          },
+              },
-          {
+              "name": "temperature",
-            name: 'temperature',
+              "state": Object {
-            state: {
+                "displayName": "even temperature",
-              displayName: 'temperature even',
+              },
+              "type": "number",
+              "values": Array [
+                undefined,
+                10.3,
+                10.4,
+                10.5,
+                10.6,
+                undefined,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'even' },
+                "name": "even",
-          },
+              },
-          {
+              "name": "humidity",
-            name: 'humidity',
+              "state": Object {
-            state: {
+                "displayName": "even humidity",
-              displayName: 'humidity even',
+              },
+              "type": "number",
+              "values": Array [
+                undefined,
+                10000.3,
+                10000.4,
+                10000.5,
+                10000.6,
+                undefined,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'even' },
+                "name": "odd",
-          },
+              },
-          {
+              "name": "temperature",
-            name: 'temperature',
+              "state": Object {
-            state: {
+                "displayName": "odd temperature",
-              displayName: 'temperature odd',
+              },
+              "type": "number",
+              "values": Array [
+                11.1,
+                11.3,
+                undefined,
+                11.5,
+                undefined,
+                11.7,
+              ],
             },
-            type: FieldType.number,
+            Object {
-            values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
+              "config": Object {},
-            config: {},
+              "labels": Object {
-            labels: { name: 'odd' },
+                "name": "odd",
-          },
+              },
-          {
+              "name": "humidity",
-            name: 'humidity',
+              "state": Object {
-            state: {
+                "displayName": "odd humidity",
-              displayName: 'humidity odd',
+              },
+              "type": "number",
+              "values": Array [
+                11000.1,
+                11000.3,
+                undefined,
+                11000.5,
+                undefined,
+                11000.7,
+              ],
             },
-            type: FieldType.number,
+          ]
-            values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
+        `);
-            config: {},
-            labels: { name: 'odd' },
-          },
-        ]);
       }
     );
   });
@@ -270,40 +306,58 @@ describe('SeriesToColumns Transformer', () => {
         (received) => {
           const data = received[0];
           const filtered = data[0];
-          const expected: Field[] = [
+          expect(filtered.fields).toMatchInlineSnapshot(`
-            {
+            Array [
-              name: 'time',
+              Object {
-              state: {
+                "config": Object {},
-                displayName: 'time',
+                "name": "time",
+                "state": Object {
+                  "displayName": "time",
+                },
+                "type": "time",
+                "values": Array [
+                  1000,
+                  2000,
+                  3000,
+                  4000,
+                ],
               },
-              type: FieldType.time,
+              Object {
-              values: new ArrayVector([1000, 2000, 3000, 4000]),
+                "config": Object {},
-              config: {},
+                "labels": Object {
-              labels: undefined,
+                  "name": "temperature",
-            },
+                },
-            {
+                "name": "temperature",
-              name: 'temperature',
+                "state": Object {
-              type: FieldType.number,
+                  "displayName": "temperature temperature",
-              values: new ArrayVector([1, 3, 5, 7]),
+                },
-              config: {},
+                "type": "number",
-              state: {
+                "values": Array [
-                displayName: 'temperature temperature',
+                  1,
+                  3,
+                  5,
+                  7,
+                ],
               },
-              labels: { name: 'temperature' },
+              Object {
-            },
+                "config": Object {},
-            {
+                "labels": Object {
-              name: 'temperature',
+                  "name": "B",
-              state: {
+                },
-                displayName: 'temperature B',
+                "name": "temperature",
+                "state": Object {
+                  "displayName": "B temperature",
+                },
+                "type": "number",
+                "values": Array [
+                  2,
+                  4,
+                  6,
+                  8,
+                ],
               },
-              type: FieldType.number,
+            ]
-              values: new ArrayVector([2, 4, 6, 8]),
+          `);
-              config: {},
-              labels: { name: 'B' },
-            },
-          ];
-
-          expect(filtered.fields).toEqual(expected);
         }
       );
     });
@@ -341,31 +395,55 @@ describe('SeriesToColumns Transformer', () => {
     await expect(transformDataFrame([cfg], [frame1, frame2, frame3])).toEmitValuesWith((received) => {
       const data = received[0];
       const filtered = data[0];
-      expect(filtered.fields).toEqual([
+      expect(filtered.fields).toMatchInlineSnapshot(`
-        {
+        Array [
-          name: 'time',
+          Object {
-          state: { displayName: 'time' },
+            "config": Object {},
-          type: FieldType.time,
+            "name": "time",
-          values: new ArrayVector([1, 2, 3]),
+            "state": Object {
-          config: {},
+              "displayName": "time",
-        },
+            },
-        {
+            "type": "time",
-          name: 'temperature',
+            "values": Array [
-          state: { displayName: 'temperature A' },
+              1,
-          type: FieldType.number,
+              2,
-          values: new ArrayVector([10, 11, 12]),
+              3,
-          config: {},
+            ],
-          labels: { name: 'A' },
+          },
-        },
+          Object {
-        {
+            "config": Object {},
-          name: 'temperature',
+            "labels": Object {
-          state: { displayName: 'temperature C' },
+              "name": "A",
-          type: FieldType.number,
+            },
-          values: new ArrayVector([20, 22, 24]),
+            "name": "temperature",
-          config: {},
+            "state": Object {
-          labels: { name: 'C' },
+              "displayName": "A temperature",
-        },
+            },
-      ]);
+            "type": "number",
+            "values": Array [
+              10,
+              11,
+              12,
+            ],
+          },
+          Object {
+            "config": Object {},
+            "labels": Object {
+              "name": "C",
+            },
+            "name": "temperature",
+            "state": Object {
+              "displayName": "C temperature",
+            },
+            "type": "number",
+            "values": Array [
+              20,
+              22,
+              24,
+            ],
+          },
+        ]
+      `);
     });
   });
 
@@ -394,31 +472,45 @@ describe('SeriesToColumns Transformer', () => {
     await expect(transformDataFrame([cfg], [frame1, frame2])).toEmitValuesWith((received) => {
       const data = received[0];
       const filtered = data[0];
-      expect(filtered.fields).toEqual([
+      expect(filtered.fields).toMatchInlineSnapshot(`
-        {
+        Array [
-          name: 'time',
+          Object {
-          state: { displayName: 'time' },
+            "config": Object {},
-          type: FieldType.time,
+            "name": "time",
-          values: new ArrayVector([1]),
+            "state": Object {
-          config: {},
+              "displayName": "time",
-        },
+            },
-        {
+            "type": "time",
-          name: 'temperature',
+            "values": Array [
-          state: { displayName: 'temperature 1' },
+              1,
-          type: FieldType.number,
+            ],
-          values: new ArrayVector([10]),
+          },
-          config: {},
+          Object {
-          labels: {},
+            "config": Object {},
-        },
+            "labels": Object {},
-        {
+            "name": "temperature",
-          name: 'temperature',
+            "state": Object {
-          state: { displayName: 'temperature 2' },
+              "displayName": "temperature",
-          type: FieldType.number,
+            },
-          values: new ArrayVector([20]),
+            "type": "number",
-          config: {},
+            "values": Array [
-          labels: {},
+              10,
-        },
+            ],
-      ]);
+          },
+          Object {
+            "config": Object {},
+            "labels": Object {},
+            "name": "temperature",
+            "state": Object {
+              "displayName": "temperature",
+            },
+            "type": "number",
+            "values": Array [
+              20,
+            ],
+          },
+        ]
+      `);
     });
   });
 });

packages/grafana-data/src/transformations/transformers/seriesToColumns.ts

@@ -1,153 +1,36 @@
 import { map } from 'rxjs/operators';
 
-import { DataFrame, DataTransformerInfo, Field } from '../../types';
+import { DataTransformerInfo, FieldMatcher } from '../../types';
 import { DataTransformerID } from './ids';
-import { MutableDataFrame } from '../../dataframe';
+import { outerJoinDataFrames } from './joinDataFrames';
-import { ArrayVector } from '../../vector';
+import { fieldMatchers } from '../matchers';
-import { getFieldDisplayName } from '../../field/fieldState';
+import { FieldMatcherID } from '../matchers/ids';
 
 export interface SeriesToColumnsOptions {
-  byField?: string;
+  byField?: string; // empty will pick the field automatically
 }
 
-const DEFAULT_KEY_FIELD = 'Time';
-
 export const seriesToColumnsTransformer: DataTransformerInfo<SeriesToColumnsOptions> = {
   id: DataTransformerID.seriesToColumns,
-  name: 'Series as columns',
+  name: 'Series as columns', // Called 'Outer join' in the UI!
   description: 'Groups series by field and returns values as columns',
   defaultOptions: {
-    byField: DEFAULT_KEY_FIELD,
+    byField: undefined, // DEFAULT_KEY_FIELD,
   },
   operator: (options) => (source) =>
     source.pipe(
       map((data) => {
-        return outerJoinDataFrames(data, options);
+        if (data.length > 1) {
+          let joinBy: FieldMatcher | undefined = undefined;
+          if (options.byField) {
+            joinBy = fieldMatchers.get(FieldMatcherID.byName).get(options.byField);
+          }
+          const joined = outerJoinDataFrames({ frames: data, joinBy });
+          if (joined) {
+            return [joined];
+          }
+        }
+        return data;
       })
     ),
 };
-
-/**
- * @internal
- */
-export function outerJoinDataFrames(data: DataFrame[], options: SeriesToColumnsOptions) {
-  const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD;
-  const allFields: FieldsToProcess[] = [];
-
-  for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
-    const frame = data[frameIndex];
-    const keyField = findKeyField(frame, keyFieldMatch);
-
-    if (!keyField) {
-      continue;
-    }
-
-    for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
-      const sourceField = frame.fields[fieldIndex];
-
-      if (sourceField === keyField) {
-        continue;
-      }
-
-      let labels = sourceField.labels ?? {};
-
-      if (frame.name) {
-        labels = { ...labels, name: frame.name };
-      }
-
-      allFields.push({
-        keyField,
-        sourceField,
-        newField: {
-          ...sourceField,
-          state: null,
-          values: new ArrayVector([]),
-          labels,
-        },
-      });
-    }
-  }
-
-  // if no key fields or more than one value field
-  if (allFields.length <= 1) {
-    return data;
-  }
-
-  const resultFrame = new MutableDataFrame();
-
-  resultFrame.addField({
-    ...allFields[0].keyField,
-    values: new ArrayVector([]),
-  });
-
-  for (const item of allFields) {
-    item.newField = resultFrame.addField(item.newField);
-  }
-
-  const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame);
-  const byKeyField: { [key: string]: { [key: string]: any } } = {};
-
-  /*
-    this loop creates a dictionary object that groups the key fields values
-    {
-      "key field first value as string" : {
-        "key field name": key field first value,
-        "other series name": other series value
-        "other series n name": other series n value
-      },
-      "key field n value as string" : {
-        "key field name": key field n value,
-        "other series name": other series value
-        "other series n name": other series n value
-      }
-    }
-    */
-
-  for (let fieldIndex = 0; fieldIndex < allFields.length; fieldIndex++) {
-    const { sourceField, keyField, newField } = allFields[fieldIndex];
-    const newFieldTitle = getFieldDisplayName(newField, resultFrame);
-
-    for (let valueIndex = 0; valueIndex < sourceField.values.length; valueIndex++) {
-      const value = sourceField.values.get(valueIndex);
-      const keyValue = keyField.values.get(valueIndex);
-
-      if (!byKeyField[keyValue]) {
-        byKeyField[keyValue] = { [newFieldTitle]: value, [keyFieldTitle]: keyValue };
-      } else {
-        byKeyField[keyValue][newFieldTitle] = value;
-      }
-    }
-  }
-
-  const keyValueStrings = Object.keys(byKeyField);
-  for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) {
-    const keyValueAsString = keyValueStrings[rowIndex];
-
-    for (let fieldIndex = 0; fieldIndex < resultFrame.fields.length; fieldIndex++) {
-      const field = resultFrame.fields[fieldIndex];
-      const otherColumnName = getFieldDisplayName(field, resultFrame);
-      const value = byKeyField[keyValueAsString][otherColumnName] ?? null;
-      field.values.add(value);
-    }
-  }
-
-  return [resultFrame];
-}
-
-function findKeyField(frame: DataFrame, matchTitle: string): Field | null {
-  for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
-    const field = frame.fields[fieldIndex];
-
-    if (matchTitle === getFieldDisplayName(field)) {
-      return field;
-    }
-  }
-
-  return null;
-}
-
-interface FieldsToProcess {
-  newField: Field;
-  sourceField: Field;
-  keyField: Field;
-}

packages/grafana-ui/src/components/GraphNG/GraphNG.tsx

@@ -5,14 +5,16 @@ import {
   DisplayValue,
   FieldConfig,
   FieldMatcher,
+  FieldMatcherID,
+  fieldMatchers,
   fieldReducers,
   FieldType,
   formattedValueToString,
   getFieldDisplayName,
+  outerJoinDataFrames,
   reduceField,
   TimeRange,
 } from '@grafana/data';
-import { joinDataFrames } from './utils';
 import { useTheme } from '../../themes';
 import { UPlotChart } from '../uPlot/Plot';
 import { PlotProps } from '../uPlot/types';
@@ -64,7 +66,16 @@ export const GraphNG: React.FC<GraphNGProps> = ({
   const theme = useTheme();
   const hasLegend = useRef(legend && legend.displayMode !== LegendDisplayMode.Hidden);
 
-  const frame = useMemo(() => joinDataFrames(data, fields), [data, fields]);
+  const frame = useMemo(() => {
+    // Default to timeseries config
+    if (!fields) {
+      fields = {
+        x: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
+        y: fieldMatchers.get(FieldMatcherID.numeric).get({}),
+      };
+    }
+    return outerJoinDataFrames({ frames: data, joinBy: fields.x, keep: fields.y, keepOriginIndices: true });
+  }, [data, fields]);
 
   const compareFrames = useCallback((a?: DataFrame | null, b?: DataFrame | null) => {
     if (a && b) {
@@ -107,6 +118,7 @@ export const GraphNG: React.FC<GraphNGProps> = ({
 
     // X is the first field in the aligned frame
     const xField = frame.fields[0];
+    let seriesIndex = 0;
 
     if (xField.type === FieldType.time) {
       builder.addScale({
@@ -150,6 +162,7 @@ export const GraphNG: React.FC<GraphNGProps> = ({
       if (field === xField || field.type !== FieldType.number) {
         continue;
       }
+      field.state!.seriesIndex = seriesIndex++;
 
       const fmt = field.display ?? defaultFormatter;
       const scaleKey = config.unit || FIXED_UNIT;

packages/grafana-ui/src/components/GraphNG/utils.test.ts

@@ -1,334 +0,0 @@
-import { ArrayVector, DataFrame, FieldType, toDataFrame } from '@grafana/data';
-import { joinDataFrames, isLikelyAscendingVector } from './utils';
-
-describe('joinDataFrames', () => {
-  describe('joined frame', () => {
-    it('should align multiple data frames into one data frame', () => {
-      const data: DataFrame[] = [
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature A', type: FieldType.number, values: [1, 3, 5, 7] },
-          ],
-        }),
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
-          ],
-        }),
-      ];
-
-      const joined = joinDataFrames(data);
-
-      expect(joined?.fields).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "config": Object {},
-            "name": "time",
-            "state": Object {
-              "origin": undefined,
-            },
-            "type": "time",
-            "values": Array [
-              1000,
-              2000,
-              3000,
-              4000,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature A",
-            "state": Object {
-              "displayName": "temperature A",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 0,
-              },
-              "seriesIndex": 0,
-            },
-            "type": "number",
-            "values": Array [
-              1,
-              3,
-              5,
-              7,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature B",
-            "state": Object {
-              "displayName": "temperature B",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 1,
-              },
-              "seriesIndex": 1,
-            },
-            "type": "number",
-            "values": Array [
-              0,
-              2,
-              6,
-              7,
-            ],
-          },
-        ]
-      `);
-    });
-
-    it('should align multiple data frames into one data frame but only keep first time field', () => {
-      const data: DataFrame[] = [
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
-          ],
-        }),
-        toDataFrame({
-          fields: [
-            { name: 'time2', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
-          ],
-        }),
-      ];
-
-      const aligned = joinDataFrames(data);
-
-      expect(aligned?.fields).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "config": Object {},
-            "name": "time",
-            "state": Object {
-              "origin": undefined,
-            },
-            "type": "time",
-            "values": Array [
-              1000,
-              2000,
-              3000,
-              4000,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature",
-            "state": Object {
-              "displayName": "temperature",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 0,
-              },
-              "seriesIndex": 0,
-            },
-            "type": "number",
-            "values": Array [
-              1,
-              3,
-              5,
-              7,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature B",
-            "state": Object {
-              "displayName": "temperature B",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 1,
-              },
-              "seriesIndex": 1,
-            },
-            "type": "number",
-            "values": Array [
-              0,
-              2,
-              6,
-              7,
-            ],
-          },
-        ]
-      `);
-    });
-
-    it('should align multiple data frames into one data frame and skip non-numeric fields', () => {
-      const data: DataFrame[] = [
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
-            { name: 'state', type: FieldType.string, values: ['on', 'off', 'off', 'on'] },
-          ],
-        }),
-      ];
-
-      const aligned = joinDataFrames(data);
-
-      expect(aligned?.fields).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "config": Object {},
-            "name": "time",
-            "state": Object {
-              "origin": undefined,
-            },
-            "type": "time",
-            "values": Array [
-              1000,
-              2000,
-              3000,
-              4000,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature",
-            "state": Object {
-              "displayName": "temperature",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 0,
-              },
-              "seriesIndex": 0,
-            },
-            "type": "number",
-            "values": Array [
-              1,
-              3,
-              5,
-              7,
-            ],
-          },
-        ]
-      `);
-    });
-
-    it('should align multiple data frames into one data frame and skip non-numeric fields', () => {
-      const data: DataFrame[] = [
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
-            { name: 'state', type: FieldType.string, values: ['on', 'off', 'off', 'on'] },
-          ],
-        }),
-      ];
-
-      const aligned = joinDataFrames(data);
-
-      expect(aligned?.fields).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "config": Object {},
-            "name": "time",
-            "state": Object {
-              "origin": undefined,
-            },
-            "type": "time",
-            "values": Array [
-              1000,
-              2000,
-              3000,
-              4000,
-            ],
-          },
-          Object {
-            "config": Object {},
-            "name": "temperature",
-            "state": Object {
-              "displayName": "temperature",
-              "origin": Object {
-                "fieldIndex": 1,
-                "frameIndex": 0,
-              },
-              "seriesIndex": 0,
-            },
-            "type": "number",
-            "values": Array [
-              1,
-              3,
-              5,
-              7,
-            ],
-          },
-        ]
-      `);
-    });
-  });
-
-  describe('getDataFrameFieldIndex', () => {
-    let aligned: DataFrame | null;
-
-    beforeAll(() => {
-      const data: DataFrame[] = [
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature A', type: FieldType.number, values: [1, 3, 5, 7] },
-          ],
-        }),
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
-            { name: 'humidity', type: FieldType.number, values: [0, 2, 6, 7] },
-          ],
-        }),
-        toDataFrame({
-          fields: [
-            { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
-            { name: 'temperature C', type: FieldType.number, values: [0, 2, 6, 7] },
-          ],
-        }),
-      ];
-
-      aligned = joinDataFrames(data);
-    });
-
-    it.each`
-      yDim | index
-      ${1} | ${[0, 1]}
-      ${2} | ${[1, 1]}
-      ${3} | ${[1, 2]}
-      ${4} | ${[2, 1]}
-    `('should return correct index for yDim', ({ yDim, index }) => {
-      const [frameIndex, fieldIndex] = index;
-
-      expect(aligned?.fields[yDim].state?.origin).toEqual({
-        frameIndex,
-        fieldIndex,
-      });
-    });
-  });
-
-  describe('check ascending data', () => {
-    it('simple ascending', () => {
-      const v = new ArrayVector([1, 2, 3, 4, 5]);
-      expect(isLikelyAscendingVector(v)).toBeTruthy();
-    });
-    it('simple ascending with null', () => {
-      const v = new ArrayVector([null, 2, 3, 4, null]);
-      expect(isLikelyAscendingVector(v)).toBeTruthy();
-    });
-    it('single value', () => {
-      const v = new ArrayVector([null, null, null, 4, null]);
-      expect(isLikelyAscendingVector(v)).toBeTruthy();
-      expect(isLikelyAscendingVector(new ArrayVector([4]))).toBeTruthy();
-      expect(isLikelyAscendingVector(new ArrayVector([]))).toBeTruthy();
-    });
-
-    it('middle values', () => {
-      const v = new ArrayVector([null, null, 5, 4, null]);
-      expect(isLikelyAscendingVector(v)).toBeFalsy();
-    });
-
-    it('decending', () => {
-      expect(isLikelyAscendingVector(new ArrayVector([7, 6, null]))).toBeFalsy();
-      expect(isLikelyAscendingVector(new ArrayVector([7, 8, 6]))).toBeFalsy();
-    });
-  });
-});

packages/grafana-ui/src/components/GraphNG/utils.ts

@@ -1,180 +0,0 @@
-import {
-  DataFrame,
-  ArrayVector,
-  NullValueMode,
-  getFieldDisplayName,
-  Field,
-  fieldMatchers,
-  FieldMatcherID,
-  FieldType,
-  FieldState,
-  DataFrameFieldIndex,
-  sortDataFrame,
-  Vector,
-} from '@grafana/data';
-import uPlot, { AlignedData, JoinNullMode } from 'uplot';
-import { XYFieldMatchers } from './GraphNG';
-
-// the results ofter passing though data
-export interface XYDimensionFields {
-  x: Field; // independent axis (cause)
-  y: Field[]; // dependent axis (effect)
-}
-
-export function mapDimesions(match: XYFieldMatchers, frame: DataFrame, frames?: DataFrame[]): XYDimensionFields {
-  let x: Field | undefined;
-  const y: Field[] = [];
-
-  for (const field of frame.fields) {
-    if (!x && match.x(field, frame, frames ?? [])) {
-      x = field;
-    }
-    if (match.y(field, frame, frames ?? [])) {
-      y.push(field);
-    }
-  }
-  return { x: x as Field, y };
-}
-
-/**
- * Returns a single DataFrame with:
- * - A shared time column
- * - only numeric fields
- *
- * @alpha
- */
-export function joinDataFrames(frames: DataFrame[], fields?: XYFieldMatchers): DataFrame | null {
-  const valuesFromFrames: AlignedData[] = [];
-  const sourceFields: Field[] = [];
-  const sourceFieldsRefs: Record<number, DataFrameFieldIndex> = {};
-  const nullModes: JoinNullMode[][] = [];
-
-  // Default to timeseries config
-  if (!fields) {
-    fields = {
-      x: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
-      y: fieldMatchers.get(FieldMatcherID.numeric).get({}),
-    };
-  }
-
-  for (let frameIndex = 0; frameIndex < frames.length; frameIndex++) {
-    let frame = frames[frameIndex];
-    let dims = mapDimesions(fields, frame, frames);
-
-    if (!(dims.x && dims.y.length)) {
-      continue; // no numeric and no time fields
-    }
-
-    // Quick check that x is ascending order
-    if (!isLikelyAscendingVector(dims.x.values)) {
-      const xIndex = frame.fields.indexOf(dims.x);
-      frame = sortDataFrame(frame, xIndex);
-      dims = mapDimesions(fields, frame, frames);
-    }
-
-    let nullModesFrame: JoinNullMode[] = [0];
-
-    // Add the first X axis
-    if (!sourceFields.length) {
-      sourceFields.push(dims.x);
-    }
-
-    const alignedData: AlignedData = [
-      dims.x.values.toArray(), // The x axis (time)
-    ];
-
-    for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
-      const field = frame.fields[fieldIndex];
-
-      if (!fields.y(field, frame, frames)) {
-        continue;
-      }
-
-      let values = field.values.toArray();
-      let joinNullMode = field.config.custom?.spanNulls ? 0 : 2;
-
-      if (field.config.nullValueMode === NullValueMode.AsZero) {
-        values = values.map((v) => (v === null ? 0 : v));
-        joinNullMode = 0;
-      }
-
-      sourceFieldsRefs[sourceFields.length] = { frameIndex, fieldIndex };
-
-      alignedData.push(values);
-      nullModesFrame.push(joinNullMode);
-
-      // This will cache an appropriate field name in the field state
-      getFieldDisplayName(field, frame, frames);
-      sourceFields.push(field);
-    }
-
-    valuesFromFrames.push(alignedData);
-    nullModes.push(nullModesFrame);
-  }
-
-  if (valuesFromFrames.length === 0) {
-    return null;
-  }
-
-  // do the actual alignment (outerJoin on the first arrays)
-  let joinedData = uPlot.join(valuesFromFrames, nullModes);
-
-  if (joinedData!.length !== sourceFields.length) {
-    throw new Error('outerJoinValues lost a field?');
-  }
-
-  let seriesIdx = 0;
-  // Replace the values from the outer-join field
-  return {
-    ...frames[0],
-    length: joinedData![0].length,
-    fields: joinedData!.map((vals, idx) => {
-      let state: FieldState = {
-        ...sourceFields[idx].state,
-        origin: sourceFieldsRefs[idx],
-      };
-
-      if (sourceFields[idx].type !== FieldType.time) {
-        state.seriesIndex = seriesIdx;
-        seriesIdx++;
-      }
-
-      return {
-        ...sourceFields[idx],
-        state,
-        values: new ArrayVector(vals),
-      };
-    }),
-  };
-}
-
-// Quick test if the first and last points look to be ascending
-export function isLikelyAscendingVector(data: Vector): boolean {
-  let first: any = undefined;
-
-  for (let idx = 0; idx < data.length; idx++) {
-    const v = data.get(idx);
-    if (v != null) {
-      if (first != null) {
-        if (first > v) {
-          return false; // descending
-        }
-        break;
-      }
-      first = v;
-    }
-  }
-
-  let idx = data.length - 1;
-  while (idx >= 0) {
-    const v = data.get(idx--);
-    if (v != null) {
-      if (first > v) {
-        return false;
-      }
-      return true;
-    }
-  }
-
-  return true; // only one non-null point
-}