From de1b2bdc4c72611e99d530f25170a28bac5ba328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Tue, 6 Oct 2020 07:55:09 +0200 Subject: [PATCH] Transformations: Use Observable (#27584) * WIP: changes transformer API to Observable * Refactor: changes ResultProcessor * WIP: TransformationUI * wip * Refactor: Cleaning up code in CalculateFieldTransformerEditorProps * Refactor: pushing editor and input output down to TransformationEditor * Refactor: renaming props * Refactor: makes transformDataFrame more readable * Refactor: fixes editor issues * Refactor: changes after merge with master and first tests * Tests: completes the tests for the Explore changes * Tests: fixes all transform tests * Tests: fixed annotations test * Tests: fixed typechecks * Refactor: changes transform interface to MonoTypeOperatorFunction * Chore: cleans up typings * Update packages/grafana-data/src/transformations/transformDataFrame.ts Co-authored-by: Marcus Andersson * Tests: fixes broken tests Co-authored-by: Marcus Andersson --- .../src/transformations/transformDataFrame.ts | 71 ++- .../transformers/append.test.ts | 26 +- .../transformations/transformers/append.ts | 84 +-- .../transformers/calculateField.test.ts | 189 ++++--- .../transformers/calculateField.ts | 106 ++-- .../transformers/ensureColumns.test.ts | 36 +- .../transformers/ensureColumns.ts | 28 +- .../transformers/filter.test.ts | 15 +- .../transformations/transformers/filter.ts | 98 ++-- .../transformers/filterByName.test.ts | 139 +++-- .../transformers/filterByName.ts | 17 +- .../transformers/filterByRefId.test.ts | 24 +- .../transformers/filterByRefId.ts | 6 +- .../transformers/groupBy.test.ts | 208 ++++--- .../transformations/transformers/groupBy.ts | 213 +++---- .../transformers/labelsToFields.test.ts | 93 +-- .../transformers/labelsToFields.ts | 94 +-- .../transformers/merge.test.ts | 393 +++++++------ .../src/transformations/transformers/merge.ts | 157 ++--- .../src/transformations/transformers/noop.ts | 5 +- .../transformers/order.test.ts | 196 ++++--- .../src/transformations/transformers/order.ts | 26 +- .../transformers/organize.test.ts | 115 ++-- .../transformations/transformers/organize.ts | 22 +- .../transformers/reduce.test.ts | 321 ++++++----- .../transformations/transformers/reduce.ts | 146 ++--- .../transformers/rename.test.ts | 218 +++---- .../transformations/transformers/rename.ts | 26 +- .../transformers/seriesToColumns.test.ts | 531 +++++++++-------- .../transformers/seriesToColumns.ts | 145 ++--- .../transformers/seriesToRows.test.ts | 139 +++-- .../transformers/seriesToRows.ts | 100 ++-- .../grafana-data/src/types/annotations.ts | 6 +- .../grafana-data/src/types/transformations.ts | 6 +- packages/grafana-data/src/utils/index.ts | 1 + .../src/utils/tests}/observableTester.ts | 5 +- .../CalculateFieldTransformerEditor.tsx | 115 ++-- .../features/annotations/annotations_srv.ts | 17 +- .../standardAnnotationSupport.test.ts | 159 +++--- .../annotations/standardAnnotationSupport.ts | 201 ++++--- .../components/Inspector/InspectDataTab.tsx | 78 ++- .../TransformationEditor.tsx | 77 ++- .../TransformationOperationRow.tsx | 34 +- .../TransformationOperationRows.tsx | 43 ++ .../TransformationsEditor.tsx | 62 +- .../components/TransformationsEditor/types.ts | 6 + .../dashboard/state/PanelQueryRunner.ts | 56 +- .../app/features/explore/state/actionTypes.ts | 7 +- public/app/features/explore/state/actions.ts | 38 +- public/app/features/explore/state/reducers.ts | 22 +- .../explore/utils/ResultProcessor.test.ts | 323 ----------- .../features/explore/utils/ResultProcessor.ts | 161 ------ .../features/explore/utils/decorators.test.ts | 534 ++++++++++++++++++ .../app/features/explore/utils/decorators.ts | 214 +++++++ .../datasource/loki/datasource.test.ts | 2 +- .../postgres/specs/datasource.test.ts | 3 +- .../app/plugins/panel/graph3/GraphPanel.tsx | 18 +- public/app/plugins/panel/graph3/utils.ts | 17 +- public/app/types/explore.ts | 34 +- 59 files changed, 3540 insertions(+), 2686 deletions(-) rename {public/test/helpers => packages/grafana-data/src/utils/tests}/observableTester.ts (96%) create mode 100644 public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRows.tsx create mode 100644 public/app/features/dashboard/components/TransformationsEditor/types.ts delete mode 100644 public/app/features/explore/utils/ResultProcessor.test.ts delete mode 100644 public/app/features/explore/utils/ResultProcessor.ts create mode 100644 public/app/features/explore/utils/decorators.test.ts create mode 100644 public/app/features/explore/utils/decorators.ts diff --git a/packages/grafana-data/src/transformations/transformDataFrame.ts b/packages/grafana-data/src/transformations/transformDataFrame.ts index 4e51c2f2dc1..9ed30613ae9 100644 --- a/packages/grafana-data/src/transformations/transformDataFrame.ts +++ b/packages/grafana-data/src/transformations/transformDataFrame.ts @@ -1,37 +1,68 @@ +import { MonoTypeOperatorFunction, Observable, of } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; + import { DataFrame, DataTransformerConfig } from '../types'; -import { standardTransformersRegistry } from './standardTransformersRegistry'; +import { standardTransformersRegistry, TransformerRegistyItem } from './standardTransformersRegistry'; -/** - * Apply configured transformations to the input data - */ -export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] { - let processed = data; - for (const config of options) { - const info = standardTransformersRegistry.get(config.id); +const getOperator = (config: DataTransformerConfig): MonoTypeOperatorFunction => source => { + const info = standardTransformersRegistry.get(config.id); - if (!info) { - return data; - } + if (!info) { + return source; + } - const defaultOptions = info.transformation.defaultOptions ?? {}; - const options = { ...defaultOptions, ...config.options }; - const transformer = info.transformation.transformer(options); - const after = transformer(processed); + const defaultOptions = info.transformation.defaultOptions ?? {}; + const options = { ...defaultOptions, ...config.options }; - // Add a key to the metadata if the data changed - if (after && after !== processed) { + return source.pipe( + mergeMap(before => of(before).pipe(info.transformation.operator(options), postProcessTransform(before, info))) + ); +}; + +const postProcessTransform = ( + before: DataFrame[], + info: TransformerRegistyItem +): MonoTypeOperatorFunction => source => + source.pipe( + map(after => { + if (after === before) { + return after; + } + + // Add a key to the metadata if the data changed for (const series of after) { if (!series.meta) { series.meta = {}; } + if (!series.meta.transformations) { series.meta.transformations = [info.id]; } else { series.meta.transformations = [...series.meta.transformations, info.id]; } } - processed = after; - } + + return after; + }) + ); + +/** + * Apply configured transformations to the input data + */ +export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): Observable { + const stream = of(data); + + if (!options.length) { + return stream; } - return processed; + + const operators: Array> = []; + + for (let index = 0; index < options.length; index++) { + const config = options[index]; + operators.push(getOperator(config)); + } + + // @ts-ignore TypeScript has a hard time understanding this construct + return stream.pipe.apply(stream, operators); } diff --git a/packages/grafana-data/src/transformations/transformers/append.test.ts b/packages/grafana-data/src/transformations/transformers/append.test.ts index 1c2a07e3008..afc35a63783 100644 --- a/packages/grafana-data/src/transformations/transformers/append.test.ts +++ b/packages/grafana-data/src/transformations/transformers/append.test.ts @@ -3,6 +3,7 @@ import { toDataFrame } from '../../dataframe/processDataFrame'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { appendTransformer } from './append'; import { transformDataFrame } from '../transformDataFrame'; +import { observableTester } from '../../utils/tests/observableTester'; const seriesAB = toDataFrame({ columns: [{ text: 'A' }, { text: 'B' }], @@ -24,21 +25,28 @@ describe('Append Transformer', () => { beforeAll(() => { mockTransformationsRegistry([appendTransformer]); }); - it('filters by include', () => { + + it('filters by include', done => { const cfg = { id: DataTransformerID.append, options: {}, }; - const processed = transformDataFrame([cfg], [seriesAB, seriesBC])[0]; - expect(processed.fields.length).toBe(3); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesAB, seriesBC]), + expect: data => { + const processed = data[0]; + expect(processed.fields.length).toBe(3); - const fieldA = processed.fields[0]; - const fieldB = processed.fields[1]; - const fieldC = processed.fields[2]; + const fieldA = processed.fields[0]; + const fieldB = processed.fields[1]; + const fieldC = processed.fields[2]; - expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]); - expect(fieldB.values.toArray()).toEqual([100, 200, null, null]); - expect(fieldC.values.toArray()).toEqual([null, null, 3000, 4000]); + expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]); + expect(fieldB.values.toArray()).toEqual([100, 200, null, null]); + expect(fieldC.values.toArray()).toEqual([null, null, 3000, 4000]); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/append.ts b/packages/grafana-data/src/transformations/transformers/append.ts index 952f006bbe2..69d3459f6bb 100644 --- a/packages/grafana-data/src/transformations/transformers/append.ts +++ b/packages/grafana-data/src/transformations/transformers/append.ts @@ -1,4 +1,5 @@ -import { DataFrame } from '../../types/dataFrame'; +import { map } from 'rxjs/operators'; + import { DataTransformerID } from './ids'; import { MutableDataFrame } from '../../dataframe/MutableDataFrame'; import { DataTransformerInfo } from '../../types/transformations'; @@ -15,45 +16,46 @@ export const appendTransformer: DataTransformerInfo = { * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: AppendOptions) => { - return (data: DataFrame[]) => { - if (data.length < 2) { - return data; - } - - // Add the first row - const processed = new MutableDataFrame(); - for (const f of data[0].fields) { - processed.addField({ - ...f, - values: [...f.values.toArray()], - }); - } - - for (let i = 1; i < data.length; i++) { - const frame = data[i]; - const startLength = frame.length; - for (let j = 0; j < frame.fields.length; j++) { - const src = frame.fields[j]; - let vals = processed.values[src.name]; - if (!vals) { - vals = processed.addField( - { - ...src, - values: [], - }, - startLength - ).values; - } - - // Add each row - for (let k = 0; k < frame.length; k++) { - vals.add(src.values.get(k)); - } + operator: options => source => + source.pipe( + map(data => { + if (data.length < 2) { + return data; } - processed.validate(); - } - return [processed]; - }; - }, + + // Add the first row + const processed = new MutableDataFrame(); + for (const f of data[0].fields) { + processed.addField({ + ...f, + values: [...f.values.toArray()], + }); + } + + for (let i = 1; i < data.length; i++) { + const frame = data[i]; + const startLength = frame.length; + for (let j = 0; j < frame.fields.length; j++) { + const src = frame.fields[j]; + let vals = processed.values[src.name]; + if (!vals) { + vals = processed.addField( + { + ...src, + values: [], + }, + startLength + ).values; + } + + // Add each row + for (let k = 0; k < frame.length; k++) { + vals.add(src.values.get(k)); + } + } + processed.validate(); + } + return [processed]; + }) + ), }; diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.test.ts b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts index 7cf684c2aa1..13fd0bc80ed 100644 --- a/packages/grafana-data/src/transformations/transformers/calculateField.test.ts +++ b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts @@ -4,9 +4,10 @@ import { FieldType } from '../../types/dataFrame'; import { ReducerID } from '../fieldReducer'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { transformDataFrame } from '../transformDataFrame'; -import { calculateFieldTransformer, CalculateFieldMode, ReduceOptions } from './calculateField'; +import { CalculateFieldMode, calculateFieldTransformer, ReduceOptions } from './calculateField'; import { DataFrameView } from '../../dataframe'; import { BinaryOperationID } from '../../utils'; +import { observableTester } from '../../utils/tests/observableTester'; const seriesA = toDataFrame({ fields: [ @@ -29,7 +30,7 @@ describe('calculateField transformer w/ timeseries', () => { mockTransformationsRegistry([calculateFieldTransformer]); }); - it('will filter and alias', () => { + it('will filter and alias', done => { const cfg = { id: DataTransformerID.calculateField, options: { @@ -38,31 +39,35 @@ describe('calculateField transformer w/ timeseries', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesA, seriesBC])[0]; - const rows = new DataFrameView(filtered).toArray(); - expect(rows).toMatchInlineSnapshot(` - Array [ - Object { - "A": 1, - "B": 2, - "C": 3, - "D": "first", - "The Total": 6, - "TheTime": 1000, - }, - Object { - "A": 100, - "B": 200, - "C": 300, - "D": "second", - "The Total": 600, - "TheTime": 2000, - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesBC]), + expect: data => { + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + A: 1, + B: 2, + C: 3, + D: 'first', + 'The Total': 6, + TheTime: 1000, + }, + { + A: 100, + B: 200, + C: 300, + D: 'second', + 'The Total': 600, + TheTime: 2000, + }, + ]); + }, + done, + }); }); - it('will replace other fields', () => { + it('will replace other fields', done => { const cfg = { id: DataTransformerID.calculateField, options: { @@ -74,23 +79,27 @@ describe('calculateField transformer w/ timeseries', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesA, seriesBC])[0]; - const rows = new DataFrameView(filtered).toArray(); - expect(rows).toMatchInlineSnapshot(` - Array [ - Object { - "Mean": 2, - "TheTime": 1000, - }, - Object { - "Mean": 200, - "TheTime": 2000, - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesBC]), + expect: data => { + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + Mean: 2, + TheTime: 1000, + }, + { + Mean: 200, + TheTime: 2000, + }, + ]); + }, + done, + }); }); - it('will filter by name', () => { + it('will filter by name', done => { const cfg = { id: DataTransformerID.calculateField, options: { @@ -103,23 +112,27 @@ describe('calculateField transformer w/ timeseries', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesBC])[0]; - const rows = new DataFrameView(filtered).toArray(); - expect(rows).toMatchInlineSnapshot(` - Array [ - Object { - "Mean": 2, - "TheTime": 1000, - }, - Object { - "Mean": 200, - "TheTime": 2000, - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesBC]), + expect: data => { + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + Mean: 2, + TheTime: 1000, + }, + { + Mean: 200, + TheTime: 2000, + }, + ]); + }, + done, + }); }); - it('binary math', () => { + it('binary math', done => { const cfg = { id: DataTransformerID.calculateField, options: { @@ -133,23 +146,27 @@ describe('calculateField transformer w/ timeseries', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesBC])[0]; - const rows = new DataFrameView(filtered).toArray(); - expect(rows).toMatchInlineSnapshot(` - Array [ - Object { - "B + C": 5, - "TheTime": 1000, - }, - Object { - "B + C": 500, - "TheTime": 2000, - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesBC]), + expect: data => { + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + 'B + C': 5, + TheTime: 1000, + }, + { + 'B + C': 500, + TheTime: 2000, + }, + ]); + }, + done, + }); }); - it('field + static number', () => { + it('field + static number', done => { const cfg = { id: DataTransformerID.calculateField, options: { @@ -163,19 +180,23 @@ describe('calculateField transformer w/ timeseries', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesBC])[0]; - const rows = new DataFrameView(filtered).toArray(); - expect(rows).toMatchInlineSnapshot(` - Array [ - Object { - "B + 2": 4, - "TheTime": 1000, - }, - Object { - "B + 2": 202, - "TheTime": 2000, - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesBC]), + expect: data => { + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + 'B + 2': 4, + TheTime: 1000, + }, + { + 'B + 2': 202, + TheTime: 2000, + }, + ]); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.ts b/packages/grafana-data/src/transformations/transformers/calculateField.ts index 919119b2f43..4d48c100f85 100644 --- a/packages/grafana-data/src/transformations/transformers/calculateField.ts +++ b/packages/grafana-data/src/transformations/transformers/calculateField.ts @@ -1,16 +1,18 @@ -import { DataFrame, DataTransformerInfo, Vector, FieldType, Field, NullValueMode } from '../../types'; +import { map } from 'rxjs/operators'; + +import { DataFrame, DataTransformerInfo, Field, FieldType, NullValueMode, Vector } from '../../types'; import { DataTransformerID } from './ids'; -import { ReducerID, fieldReducers } from '../fieldReducer'; +import { doStandardCalcs, fieldReducers, ReducerID } from '../fieldReducer'; import { getFieldMatcher } from '../matchers'; import { FieldMatcherID } from '../matchers/ids'; import { RowVector } from '../../vector/RowVector'; import { ArrayVector, BinaryOperationVector, ConstantVector } from '../../vector'; -import { doStandardCalcs } from '../fieldReducer'; import { getTimeField } from '../../dataframe/processDataFrame'; import defaults from 'lodash/defaults'; import { BinaryOperationID, binaryOperators } from '../../utils/binaryOperators'; import { ensureColumnsTransformer } from './ensureColumns'; import { getFieldDisplayName } from '../../field'; +import { noopTransformer } from './noop'; export enum CalculateFieldMode { ReduceRow = 'reduceRow', @@ -68,56 +70,60 @@ export const calculateFieldTransformer: DataTransformerInfo (data: DataFrame[]) => { - if (options && options.timeSeries !== false) { - data = ensureColumnsTransformer.transformer(null)(data); - } + operator: options => outerSource => { + const operator = + options && options.timeSeries !== false ? ensureColumnsTransformer.operator(null) : noopTransformer.operator({}); - const mode = options.mode ?? CalculateFieldMode.ReduceRow; - let creator: ValuesCreator | undefined = undefined; + return outerSource.pipe( + operator, + map(data => { + const mode = options.mode ?? CalculateFieldMode.ReduceRow; + let creator: ValuesCreator | undefined = undefined; - if (mode === CalculateFieldMode.ReduceRow) { - creator = getReduceRowCreator(defaults(options.reduce, defaultReduceOptions), data); - } else if (mode === CalculateFieldMode.BinaryOperation) { - creator = getBinaryCreator(defaults(options.binary, defaultBinaryOptions), data); - } - - // Nothing configured - if (!creator) { - return data; - } - - return data.map(frame => { - // delegate field creation to the specific function - const values = creator!(frame); - if (!values) { - return frame; - } - - const field = { - name: getNameFromOptions(options), - type: FieldType.number, - config: {}, - values, - }; - let fields: Field[] = []; - - // Replace all fields with the single field - if (options.replaceFields) { - const { timeField } = getTimeField(frame); - if (timeField && options.timeSeries !== false) { - fields = [timeField, field]; - } else { - fields = [field]; + if (mode === CalculateFieldMode.ReduceRow) { + creator = getReduceRowCreator(defaults(options.reduce, defaultReduceOptions), data); + } else if (mode === CalculateFieldMode.BinaryOperation) { + creator = getBinaryCreator(defaults(options.binary, defaultBinaryOptions), data); } - } else { - fields = [...frame.fields, field]; - } - return { - ...frame, - fields, - }; - }); + + // Nothing configured + if (!creator) { + return data; + } + + return data.map(frame => { + // delegate field creation to the specific function + const values = creator!(frame); + if (!values) { + return frame; + } + + const field = { + name: getNameFromOptions(options), + type: FieldType.number, + config: {}, + values, + }; + let fields: Field[] = []; + + // Replace all fields with the single field + if (options.replaceFields) { + const { timeField } = getTimeField(frame); + if (timeField && options.timeSeries !== false) { + fields = [timeField, field]; + } else { + fields = [field]; + } + } else { + fields = [...frame.fields, field]; + } + return { + ...frame, + fields, + }; + }); + }) + ); }, }; diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts index b0301c205e7..02df078b203 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts @@ -5,6 +5,7 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio import { transformDataFrame } from '../transformDataFrame'; import { ensureColumnsTransformer } from './ensureColumns'; import { seriesToColumnsTransformer } from './seriesToColumns'; +import { observableTester } from '../../utils/tests/observableTester'; const seriesA = toDataFrame({ fields: [ @@ -35,17 +36,19 @@ describe('ensureColumns transformer', () => { mockTransformationsRegistry([ensureColumnsTransformer, seriesToColumnsTransformer]); }); - it('will transform to columns if time field exists and multiple frames', () => { + it('will transform to columns if time field exists and multiple frames', done => { const cfg = { id: DataTransformerID.ensureColumns, options: {}, }; const data = [seriesA, seriesBC]; - const filtered = transformDataFrame([cfg], data); - expect(filtered.length).toEqual(1); - expect(filtered[0]).toMatchInlineSnapshot(` + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], data), + expect: filtered => { + expect(filtered.length).toEqual(1); + expect(filtered[0]).toMatchInlineSnapshot(` Object { "fields": Array [ Object { @@ -108,29 +111,42 @@ describe('ensureColumns transformer', () => { "refId": undefined, } `); + }, + done, + }); }); - it('will not transform to columns if time field is missing for any of the series', () => { + it('will not transform to columns if time field is missing for any of the series', done => { const cfg = { id: DataTransformerID.ensureColumns, options: {}, }; const data = [seriesBC, seriesNoTime]; - const filtered = transformDataFrame([cfg], data); - expect(filtered).toEqual(data); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], data), + expect: filtered => { + expect(filtered).toEqual(data); + }, + done, + }); }); - it('will not transform to columns if only one series', () => { + it('will not transform to columns if only one series', done => { const cfg = { id: DataTransformerID.ensureColumns, options: {}, }; const data = [seriesBC]; - const filtered = transformDataFrame([cfg], data); - expect(filtered).toEqual(data); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], data), + expect: filtered => { + expect(filtered).toEqual(data); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts index a5eb8b6fdc7..2c8f23ed3d5 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts @@ -1,25 +1,33 @@ +import { of } from 'rxjs'; + import { seriesToColumnsTransformer } from './seriesToColumns'; import { DataFrame } from '../../types/dataFrame'; import { getTimeField } from '../../dataframe/processDataFrame'; import { DataTransformerInfo } from '../../types/transformations'; import { DataTransformerID } from './ids'; +import { mergeMap } from 'rxjs/operators'; export const ensureColumnsTransformer: DataTransformerInfo = { id: DataTransformerID.ensureColumns, name: 'Ensure Columns Transformer', description: 'Will check if current data frames is series or columns. If in series it will convert to columns.', - transformer: () => (data: DataFrame[]) => { - // Assume timeseries should first be joined by time - const timeFieldName = findConsistentTimeFieldName(data); + operator: (options = {}) => source => + source.pipe( + mergeMap(data => { + // Assume timeseries should first be joined by time + const timeFieldName = findConsistentTimeFieldName(data); - if (data.length > 1 && timeFieldName) { - return seriesToColumnsTransformer.transformer({ - byField: timeFieldName, - })(data); - } + if (data.length > 1 && timeFieldName) { + return of(data).pipe( + seriesToColumnsTransformer.operator({ + byField: timeFieldName, + }) + ); + } - return data; - }, + return of(data); + }) + ), }; /** diff --git a/packages/grafana-data/src/transformations/transformers/filter.test.ts b/packages/grafana-data/src/transformations/transformers/filter.test.ts index 5eb3991703b..f22af4fb879 100644 --- a/packages/grafana-data/src/transformations/transformers/filter.test.ts +++ b/packages/grafana-data/src/transformations/transformers/filter.test.ts @@ -5,6 +5,7 @@ import { FieldMatcherID } from '../matchers/ids'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { filterFieldsTransformer } from './filter'; import { transformDataFrame } from '../transformDataFrame'; +import { observableTester } from '../../utils/tests/observableTester'; export const simpleSeriesWithTypes = toDataFrame({ fields: [ @@ -20,7 +21,7 @@ describe('Filter Transformer', () => { mockTransformationsRegistry([filterFieldsTransformer]); }); - it('filters by include', () => { + it('filters by include', done => { const cfg = { id: DataTransformerID.filterFields, options: { @@ -28,8 +29,14 @@ describe('Filter Transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [simpleSeriesWithTypes])[0]; - expect(filtered.fields.length).toBe(1); - expect(filtered.fields[0].name).toBe('D'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [simpleSeriesWithTypes]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(1); + expect(filtered.fields[0].name).toBe('D'); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/filter.ts b/packages/grafana-data/src/transformations/transformers/filter.ts index 4eb09a9373f..7fd5cd1c9b2 100644 --- a/packages/grafana-data/src/transformations/transformers/filter.ts +++ b/packages/grafana-data/src/transformations/transformers/filter.ts @@ -1,3 +1,5 @@ +import { map } from 'rxjs/operators'; + import { noopTransformer } from './noop'; import { DataFrame, Field } from '../../types/dataFrame'; import { DataTransformerID } from './ids'; @@ -19,46 +21,48 @@ export const filterFieldsTransformer: DataTransformerInfo = { * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: FilterOptions) => { + operator: (options: FilterOptions) => source => { if (!options.include && !options.exclude) { - return noopTransformer.transformer({}); + return source.pipe(noopTransformer.operator({})); } - const include = options.include ? getFieldMatcher(options.include) : null; - const exclude = options.exclude ? getFieldMatcher(options.exclude) : null; + return source.pipe( + map(data => { + const include = options.include ? getFieldMatcher(options.include) : null; + const exclude = options.exclude ? getFieldMatcher(options.exclude) : null; - return (data: DataFrame[]) => { - const processed: DataFrame[] = []; - for (const series of data) { - // Find the matching field indexes - const fields: Field[] = []; - for (let i = 0; i < series.fields.length; i++) { - const field = series.fields[i]; + const processed: DataFrame[] = []; + for (const series of data) { + // Find the matching field indexes + const fields: Field[] = []; + for (let i = 0; i < series.fields.length; i++) { + const field = series.fields[i]; - if (exclude) { - if (exclude(field, series, data)) { - continue; + if (exclude) { + if (exclude(field, series, data)) { + continue; + } + if (!include) { + fields.push(field); + } } - if (!include) { + if (include && include(field, series, data)) { fields.push(field); } } - if (include && include(field, series, data)) { - fields.push(field); - } - } - if (!fields.length) { - continue; + if (!fields.length) { + continue; + } + const copy = { + ...series, // all the other properties + fields, // but a different set of fields + }; + processed.push(copy); } - const copy = { - ...series, // all the other properties - fields, // but a different set of fields - }; - processed.push(copy); - } - return processed; - }; + return processed; + }) + ); }, }; @@ -72,30 +76,32 @@ export const filterFramesTransformer: DataTransformerInfo = { * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: FilterOptions) => { + operator: options => source => { if (!options.include && !options.exclude) { - return noopTransformer.transformer({}); + return source.pipe(noopTransformer.operator({})); } - const include = options.include ? getFrameMatchers(options.include) : null; - const exclude = options.exclude ? getFrameMatchers(options.exclude) : null; + return source.pipe( + map(data => { + const include = options.include ? getFrameMatchers(options.include) : null; + const exclude = options.exclude ? getFrameMatchers(options.exclude) : null; - return (data: DataFrame[]) => { - const processed: DataFrame[] = []; - for (const series of data) { - if (exclude) { - if (exclude(series)) { - continue; + const processed: DataFrame[] = []; + for (const series of data) { + if (exclude) { + if (exclude(series)) { + continue; + } + if (!include) { + processed.push(series); + } } - if (!include) { + if (include && include(series)) { processed.push(series); } } - if (include && include(series)) { - processed.push(series); - } - } - return processed; - }; + return processed; + }) + ); }, }; diff --git a/packages/grafana-data/src/transformations/transformers/filterByName.test.ts b/packages/grafana-data/src/transformations/transformers/filterByName.test.ts index 90e6690d88e..4e23a58f177 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByName.test.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByName.test.ts @@ -5,6 +5,7 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio import { filterFieldsByNameTransformer } from './filterByName'; import { filterFieldsTransformer } from './filter'; import { transformDataFrame } from '../transformDataFrame'; +import { observableTester } from '../../utils/tests/observableTester'; export const seriesWithNamesToMatch = toDataFrame({ fields: [ @@ -20,18 +21,24 @@ describe('filterByName transformer', () => { mockTransformationsRegistry([filterFieldsByNameTransformer, filterFieldsTransformer]); }); - it('returns original series if no options provided', () => { + it('returns original series if no options provided', done => { const cfg = { id: DataTransformerID.filterFields, options: {}, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(4); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(4); + }, + done, + }); }); describe('respects', () => { - it('inclusion by pattern', () => { + it('inclusion by pattern', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -41,12 +48,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('startsWithA'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('startsWithA'); + }, + done, + }); }); - it('exclusion by pattern', () => { + it('exclusion by pattern', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -56,12 +69,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); - it('inclusion and exclusion by pattern', () => { + it('inclusion and exclusion by pattern', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -70,12 +89,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(1); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(1); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); - it('inclusion by names', () => { + it('inclusion by names', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -85,12 +110,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('startsWithA'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('startsWithA'); + }, + done, + }); }); - it('exclusion by names', () => { + it('exclusion by names', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -100,12 +131,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); - it('inclusion and exclusion by names', () => { + it('inclusion and exclusion by names', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -114,12 +151,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(1); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(1); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); - it('inclusion by both', () => { + it('inclusion by both', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -130,12 +173,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('startsWithA'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('startsWithA'); + }, + done, + }); }); - it('exclusion by both', () => { + it('exclusion by both', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -146,12 +195,18 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(2); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); - it('inclusion and exclusion by both', () => { + it('inclusion and exclusion by both', done => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { @@ -160,9 +215,15 @@ describe('filterByName transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0]; - expect(filtered.fields.length).toBe(1); - expect(filtered.fields[0].name).toBe('B'); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithNamesToMatch]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields.length).toBe(1); + expect(filtered.fields[0].name).toBe('B'); + }, + done, + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/filterByName.ts b/packages/grafana-data/src/transformations/transformers/filterByName.ts index ba4ce2568d6..c898996afac 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByName.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByName.ts @@ -1,7 +1,7 @@ import { DataTransformerID } from './ids'; import { DataTransformerInfo, MatcherConfig } from '../../types/transformations'; import { FieldMatcherID } from '../matchers/ids'; -import { FilterOptions, filterFieldsTransformer } from './filter'; +import { filterFieldsTransformer } from './filter'; import { RegexpOrNamesMatcherOptions } from '../matchers/nameMatcher'; export interface FilterFieldsByNameTransformerOptions { @@ -19,14 +19,13 @@ export const filterFieldsByNameTransformer: DataTransformerInfo { - const filterOptions: FilterOptions = { - include: getMatcherConfig(options.include), - exclude: getMatcherConfig(options.exclude), - }; - - return filterFieldsTransformer.transformer(filterOptions); - }, + operator: options => source => + source.pipe( + filterFieldsTransformer.operator({ + include: getMatcherConfig(options.include), + exclude: getMatcherConfig(options.exclude), + }) + ), }; const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => { diff --git a/packages/grafana-data/src/transformations/transformers/filterByRefId.test.ts b/packages/grafana-data/src/transformations/transformers/filterByRefId.test.ts index 0bec449d212..18ee3f94b3d 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByRefId.test.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByRefId.test.ts @@ -3,6 +3,7 @@ import { toDataFrame } from '../../dataframe/processDataFrame'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { filterFramesByRefIdTransformer } from './filterByRefId'; import { transformDataFrame } from '../transformDataFrame'; +import { observableTester } from '../../utils/tests/observableTester'; export const allSeries = [ toDataFrame({ @@ -23,18 +24,24 @@ describe('filterByRefId transformer', () => { beforeAll(() => { mockTransformationsRegistry([filterFramesByRefIdTransformer]); }); - it('returns all series if no options provided', () => { + + it('returns all series if no options provided', done => { const cfg = { id: DataTransformerID.filterByRefId, options: {}, }; - const filtered = transformDataFrame([cfg], allSeries); - expect(filtered.length).toBe(3); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], allSeries), + expect: filtered => { + expect(filtered.length).toBe(3); + }, + done, + }); }); describe('respects', () => { - it('inclusion', () => { + it('inclusion', done => { const cfg = { id: DataTransformerID.filterByRefId, options: { @@ -42,8 +49,13 @@ describe('filterByRefId transformer', () => { }, }; - const filtered = transformDataFrame([cfg], allSeries); - expect(filtered.map(f => f.refId)).toEqual(['A', 'B']); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], allSeries), + expect: filtered => { + expect(filtered.map(f => f.refId)).toEqual(['A', 'B']); + }, + done, + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/filterByRefId.ts b/packages/grafana-data/src/transformations/transformers/filterByRefId.ts index 59aedb08122..31058de6571 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByRefId.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByRefId.ts @@ -1,5 +1,5 @@ import { DataTransformerID } from './ids'; -import { FilterOptions, filterFramesTransformer } from './filter'; +import { filterFramesTransformer, FilterOptions } from './filter'; import { DataTransformerInfo } from '../../types/transformations'; import { FrameMatcherID } from '../matchers/ids'; @@ -18,7 +18,7 @@ export const filterFramesByRefIdTransformer: DataTransformerInfo { + operator: options => source => { const filterOptions: FilterOptions = {}; if (options.include) { filterOptions.include = { @@ -33,6 +33,6 @@ export const filterFramesByRefIdTransformer: DataTransformerInfo { beforeAll(() => { mockTransformationsRegistry([groupByTransformer]); }); - it('should not apply transformation if config is missing group by fields', () => { + it('should not apply transformation if config is missing group by fields', done => { const testSeries = toDataFrame({ name: 'A', fields: [ @@ -35,11 +36,16 @@ describe('GroupBy transformer', () => { }, }; - const result = transformDataFrame([cfg], [testSeries]); - expect(result[0]).toBe(testSeries); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [testSeries]), + expect: result => { + expect(result[0]).toBe(testSeries); + }, + done, + }); }); - it('should group values by message', () => { + it('should group values by message', done => { const testSeries = toDataFrame({ name: 'A', fields: [ @@ -61,21 +67,25 @@ describe('GroupBy transformer', () => { }, }; - const result = transformDataFrame([cfg], [testSeries]); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [testSeries]), + expect: result => { + const expected: Field[] = [ + { + name: 'message', + type: FieldType.string, + values: new ArrayVector(['one', 'two', 'three']), + config: {}, + }, + ]; - const expected: Field[] = [ - { - name: 'message', - type: FieldType.string, - values: new ArrayVector(['one', 'two', 'three']), - config: {}, + expect(result[0].fields).toEqual(expected); }, - ]; - - expect(result[0].fields).toEqual(expected); + done, + }); }); - it('should group values by message and summarize values', () => { + it('should group values by message and summarize values', done => { const testSeries = toDataFrame({ name: 'A', fields: [ @@ -101,27 +111,31 @@ describe('GroupBy transformer', () => { }, }; - const result = transformDataFrame([cfg], [testSeries]); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [testSeries]), + expect: result => { + const expected: Field[] = [ + { + name: 'message', + type: FieldType.string, + values: new ArrayVector(['one', 'two', 'three']), + config: {}, + }, + { + name: 'values (sum)', + type: FieldType.number, + values: new ArrayVector([1, 4, 9]), + config: {}, + }, + ]; - const expected: Field[] = [ - { - name: 'message', - type: FieldType.string, - values: new ArrayVector(['one', 'two', 'three']), - config: {}, + expect(result[0].fields).toEqual(expected); }, - { - name: 'values (sum)', - type: FieldType.number, - values: new ArrayVector([1, 4, 9]), - config: {}, - }, - ]; - - expect(result[0].fields).toEqual(expected); + done, + }); }); - it('should group by and compute a few calculations for each group of values', () => { + it('should group by and compute a few calculations for each group of values', done => { const testSeries = toDataFrame({ name: 'A', fields: [ @@ -151,39 +165,43 @@ describe('GroupBy transformer', () => { }, }; - const result = transformDataFrame([cfg], [testSeries]); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [testSeries]), + expect: result => { + const expected: Field[] = [ + { + name: 'message', + type: FieldType.string, + values: new ArrayVector(['one', 'two', 'three']), + config: {}, + }, + { + name: 'time (count)', + type: FieldType.number, + values: new ArrayVector([1, 2, 3]), + config: {}, + }, + { + name: 'time (last)', + type: FieldType.time, + values: new ArrayVector([3000, 5000, 8000]), + config: {}, + }, + { + name: 'values (sum)', + type: FieldType.number, + values: new ArrayVector([1, 4, 9]), + config: {}, + }, + ]; - const expected: Field[] = [ - { - name: 'message', - type: FieldType.string, - values: new ArrayVector(['one', 'two', 'three']), - config: {}, + expect(result[0].fields).toEqual(expected); }, - { - name: 'time (count)', - type: FieldType.number, - values: new ArrayVector([1, 2, 3]), - config: {}, - }, - { - name: 'time (last)', - type: FieldType.time, - values: new ArrayVector([3000, 5000, 8000]), - config: {}, - }, - { - name: 'values (sum)', - type: FieldType.number, - values: new ArrayVector([1, 4, 9]), - config: {}, - }, - ]; - - expect(result[0].fields).toEqual(expected); + done, + }); }); - it('should group values in data frames induvidually', () => { + it('should group values in data frames induvidually', done => { const testSeries = [ toDataFrame({ name: 'A', @@ -219,39 +237,43 @@ describe('GroupBy transformer', () => { }, }; - const result = transformDataFrame([cfg], testSeries); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], testSeries), + expect: result => { + const expectedA: Field[] = [ + { + name: 'message', + type: FieldType.string, + values: new ArrayVector(['one', 'two', 'three']), + config: {}, + }, + { + name: 'values (sum)', + type: FieldType.number, + values: new ArrayVector([1, 4, 9]), + config: {}, + }, + ]; - const expectedA: Field[] = [ - { - name: 'message', - type: FieldType.string, - values: new ArrayVector(['one', 'two', 'three']), - config: {}, - }, - { - name: 'values (sum)', - type: FieldType.number, - values: new ArrayVector([1, 4, 9]), - config: {}, - }, - ]; + const expectedB: Field[] = [ + { + name: 'message', + type: FieldType.string, + values: new ArrayVector(['one', 'two', 'three']), + config: {}, + }, + { + name: 'values (sum)', + type: FieldType.number, + values: new ArrayVector([0, 7, 8]), + config: {}, + }, + ]; - const expectedB: Field[] = [ - { - name: 'message', - type: FieldType.string, - values: new ArrayVector(['one', 'two', 'three']), - config: {}, + expect(result[0].fields).toEqual(expectedA); + expect(result[1].fields).toEqual(expectedB); }, - { - name: 'values (sum)', - type: FieldType.number, - values: new ArrayVector([0, 7, 8]), - config: {}, - }, - ]; - - expect(result[0].fields).toEqual(expectedA); - expect(result[1].fields).toEqual(expectedB); + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/groupBy.ts b/packages/grafana-data/src/transformations/transformers/groupBy.ts index b2a555f771a..79210942c50 100644 --- a/packages/grafana-data/src/transformations/transformers/groupBy.ts +++ b/packages/grafana-data/src/transformations/transformers/groupBy.ts @@ -1,5 +1,7 @@ +import { map } from 'rxjs/operators'; + import { DataTransformerID } from './ids'; -import { DataFrame, FieldType, Field } from '../../types/dataFrame'; +import { DataFrame, Field, FieldType } from '../../types/dataFrame'; import { DataTransformerInfo } from '../../types/transformations'; import { getFieldDisplayName } from '../../field/fieldState'; import { ArrayVector } from '../../vector/ArrayVector'; @@ -33,127 +35,128 @@ export const groupByTransformer: DataTransformerInfo * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: GroupByTransformerOptions) => { - const hasValidConfig = Object.keys(options.fields).find( - name => options.fields[name].operation === GroupByOperationID.groupBy - ); + operator: options => source => + source.pipe( + map(data => { + const hasValidConfig = Object.keys(options.fields).find( + name => options.fields[name].operation === GroupByOperationID.groupBy + ); - return (data: DataFrame[]) => { - if (!hasValidConfig) { - return data; - } - - const processed: DataFrame[] = []; - - for (const frame of data) { - const groupByFields: Field[] = []; - - for (const field of frame.fields) { - if (shouldGroupOnField(field, options)) { - groupByFields.push(field); - } + if (!hasValidConfig) { + return data; } - if (groupByFields.length === 0) { - continue; // No group by field in this frame, ignore the frame - } + const processed: DataFrame[] = []; - // Group the values by fields and groups so we can get all values for a - // group for a given field. - const valuesByGroupKey: Record> = {}; - for (let rowIndex = 0; rowIndex < frame.length; rowIndex++) { - const groupKey = String(groupByFields.map(field => field.values.get(rowIndex))); - const valuesByField = valuesByGroupKey[groupKey] ?? {}; + for (const frame of data) { + const groupByFields: Field[] = []; - if (!valuesByGroupKey[groupKey]) { - valuesByGroupKey[groupKey] = valuesByField; + for (const field of frame.fields) { + if (shouldGroupOnField(field, options)) { + groupByFields.push(field); + } } - for (let field of frame.fields) { - const fieldName = getFieldDisplayName(field); + if (groupByFields.length === 0) { + continue; // No group by field in this frame, ignore the frame + } - if (!valuesByField[fieldName]) { - valuesByField[fieldName] = { - name: fieldName, - type: field.type, - config: { ...field.config }, - values: new ArrayVector(), - }; + // Group the values by fields and groups so we can get all values for a + // group for a given field. + const valuesByGroupKey: Record> = {}; + for (let rowIndex = 0; rowIndex < frame.length; rowIndex++) { + const groupKey = String(groupByFields.map(field => field.values.get(rowIndex))); + const valuesByField = valuesByGroupKey[groupKey] ?? {}; + + if (!valuesByGroupKey[groupKey]) { + valuesByGroupKey[groupKey] = valuesByField; } - valuesByField[fieldName].values.add(field.values.get(rowIndex)); - } - } + for (let field of frame.fields) { + const fieldName = getFieldDisplayName(field); - const fields: Field[] = []; - const groupKeys = Object.keys(valuesByGroupKey); + if (!valuesByField[fieldName]) { + valuesByField[fieldName] = { + name: fieldName, + type: field.type, + config: { ...field.config }, + values: new ArrayVector(), + }; + } - for (const field of groupByFields) { - const values = new ArrayVector(); - const fieldName = getFieldDisplayName(field); - - for (let key of groupKeys) { - const valuesByField = valuesByGroupKey[key]; - values.add(valuesByField[fieldName].values.get(0)); + valuesByField[fieldName].values.add(field.values.get(rowIndex)); + } } - fields.push({ - name: field.name, - type: field.type, - config: { - ...field.config, - }, - values: values, + const fields: Field[] = []; + const groupKeys = Object.keys(valuesByGroupKey); + + for (const field of groupByFields) { + const values = new ArrayVector(); + const fieldName = getFieldDisplayName(field); + + for (let key of groupKeys) { + const valuesByField = valuesByGroupKey[key]; + values.add(valuesByField[fieldName].values.get(0)); + } + + fields.push({ + name: field.name, + type: field.type, + config: { + ...field.config, + }, + values: values, + }); + } + + // Then for each calculations configured, compute and add a new field (column) + for (const field of frame.fields) { + if (!shouldCalculateField(field, options)) { + continue; + } + + const fieldName = getFieldDisplayName(field); + const aggregations = options.fields[fieldName].aggregations; + const valuesByAggregation: Record = {}; + + for (const groupKey of groupKeys) { + const fieldWithValuesForGroup = valuesByGroupKey[groupKey][fieldName]; + const results = reduceField({ + field: fieldWithValuesForGroup, + reducers: aggregations, + }); + + for (const aggregation of aggregations) { + if (!Array.isArray(valuesByAggregation[aggregation])) { + valuesByAggregation[aggregation] = []; + } + valuesByAggregation[aggregation].push(results[aggregation]); + } + } + + for (const aggregation of aggregations) { + const aggregationField: Field = { + name: `${fieldName} (${aggregation})`, + values: new ArrayVector(valuesByAggregation[aggregation]), + type: FieldType.other, + config: {}, + }; + + aggregationField.type = detectFieldType(aggregation, field, aggregationField); + fields.push(aggregationField); + } + } + + processed.push({ + fields, + length: groupKeys.length, }); } - // Then for each calculations configured, compute and add a new field (column) - for (const field of frame.fields) { - if (!shouldCalculateField(field, options)) { - continue; - } - - const fieldName = getFieldDisplayName(field); - const aggregations = options.fields[fieldName].aggregations; - const valuesByAggregation: Record = {}; - - for (const groupKey of groupKeys) { - const fieldWithValuesForGroup = valuesByGroupKey[groupKey][fieldName]; - const results = reduceField({ - field: fieldWithValuesForGroup, - reducers: aggregations, - }); - - for (const aggregation of aggregations) { - if (!Array.isArray(valuesByAggregation[aggregation])) { - valuesByAggregation[aggregation] = []; - } - valuesByAggregation[aggregation].push(results[aggregation]); - } - } - - for (const aggregation of aggregations) { - const aggregationField: Field = { - name: `${fieldName} (${aggregation})`, - values: new ArrayVector(valuesByAggregation[aggregation]), - type: FieldType.other, - config: {}, - }; - - aggregationField.type = detectFieldType(aggregation, field, aggregationField); - fields.push(aggregationField); - } - } - - processed.push({ - fields, - length: groupKeys.length, - }); - } - - return processed; - }; - }, + return processed; + }) + ), }; const shouldGroupOnField = (field: Field, options: GroupByTransformerOptions): boolean => { diff --git a/packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts b/packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts index 34ebc22c2a8..0e57d2d5e5a 100644 --- a/packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts +++ b/packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts @@ -1,16 +1,17 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { LabelsToFieldsOptions, labelsToFieldsTransformer } from './labelsToFields'; -import { DataTransformerConfig, FieldType, FieldDTO } from '../../types'; +import { DataTransformerConfig, FieldDTO, FieldType } from '../../types'; import { DataTransformerID } from './ids'; import { toDataFrame, toDataFrameDTO } from '../../dataframe'; import { transformDataFrame } from '../transformDataFrame'; +import { observableTester } from '../../utils/tests/observableTester'; describe('Labels as Columns', () => { beforeAll(() => { mockTransformationsRegistry([labelsToFieldsTransformer]); }); - it('data frame with two labels', () => { + it('data frame with two labels', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.labelsToFields, options: {}, @@ -24,23 +25,30 @@ describe('Labels as Columns', () => { ], }); - const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]); - const expected: FieldDTO[] = [ - { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, - { - name: 'location', - type: FieldType.string, - values: ['inside', 'inside'], - config: {}, - }, - { name: 'feelsLike', type: FieldType.string, values: ['ok', 'ok'], config: {} }, - { name: 'Value', type: FieldType.number, values: [1, 2], config: {} }, - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [source]), + expect: data => { + const result = toDataFrameDTO(data[0]); - expect(result.fields).toEqual(expected); + const expected: FieldDTO[] = [ + { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, + { + name: 'location', + type: FieldType.string, + values: ['inside', 'inside'], + config: {}, + }, + { name: 'feelsLike', type: FieldType.string, values: ['ok', 'ok'], config: {} }, + { name: 'Value', type: FieldType.number, values: [1, 2], config: {} }, + ]; + + expect(result.fields).toEqual(expected); + }, + done, + }); }); - it('data frame with two labels and valueLabel option', () => { + it('data frame with two labels and valueLabel option', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.labelsToFields, options: { valueLabel: 'name' }, @@ -63,22 +71,29 @@ describe('Labels as Columns', () => { ], }); - const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]); - const expected: FieldDTO[] = [ - { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, - { - name: 'location', - type: FieldType.string, - values: ['inside', 'inside'], - config: {}, - }, - { name: 'Request', type: FieldType.number, values: [1, 2], config: {} }, - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [source]), + expect: data => { + const result = toDataFrameDTO(data[0]); - expect(result.fields).toEqual(expected); + const expected: FieldDTO[] = [ + { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, + { + name: 'location', + type: FieldType.string, + values: ['inside', 'inside'], + config: {}, + }, + { name: 'Request', type: FieldType.number, values: [1, 2], config: {} }, + ]; + + expect(result.fields).toEqual(expected); + }, + done, + }); }); - it('two data frames with 1 value and 1 label', () => { + it('two data frames with 1 value and 1 label', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.labelsToFields, options: {}, @@ -100,14 +115,20 @@ describe('Labels as Columns', () => { ], }); - const result = toDataFrameDTO(transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB])[0]); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB]), + expect: data => { + const result = toDataFrameDTO(data[0]); - const expected: FieldDTO[] = [ - { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, - { name: 'location', type: FieldType.string, values: ['inside', 'outside'], config: {} }, - { name: 'temp', type: FieldType.number, values: [1, -1], config: {} }, - ]; + const expected: FieldDTO[] = [ + { name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, + { name: 'location', type: FieldType.string, values: ['inside', 'outside'], config: {} }, + { name: 'temp', type: FieldType.number, values: [1, -1], config: {} }, + ]; - expect(result.fields).toEqual(expected); + expect(result.fields).toEqual(expected); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/labelsToFields.ts b/packages/grafana-data/src/transformations/transformers/labelsToFields.ts index abe74276016..9d983224efd 100644 --- a/packages/grafana-data/src/transformations/transformers/labelsToFields.ts +++ b/packages/grafana-data/src/transformations/transformers/labelsToFields.ts @@ -1,4 +1,6 @@ -import { DataFrame, DataTransformerInfo, FieldType, Field } from '../../types'; +import { map } from 'rxjs/operators'; + +import { DataFrame, DataTransformerInfo, Field, FieldType } from '../../types'; import { DataTransformerID } from './ids'; import { ArrayVector } from '../../vector'; import { mergeTransformer } from './merge'; @@ -15,56 +17,60 @@ export const labelsToFieldsTransformer: DataTransformerInfo (data: DataFrame[]) => { - const result: DataFrame[] = []; + operator: options => source => + source.pipe( + map(data => { + const result: DataFrame[] = []; - for (const frame of data) { - const newFields: Field[] = []; + for (const frame of data) { + const newFields: Field[] = []; - for (const field of frame.fields) { - if (!field.labels) { - newFields.push(field); - continue; - } + for (const field of frame.fields) { + if (!field.labels) { + newFields.push(field); + continue; + } - let name = field.name; + let name = field.name; - for (const labelName of Object.keys(field.labels)) { - // if we should use this label as the value field name store it and skip adding this as a seperate field - if (options.valueLabel === labelName) { - name = field.labels[labelName]; - continue; + for (const labelName of Object.keys(field.labels)) { + // if we should use this label as the value field name store it and skip adding this as a seperate field + if (options.valueLabel === labelName) { + name = field.labels[labelName]; + continue; + } + + const values = new Array(frame.length).fill(field.labels[labelName]); + newFields.push({ + name: labelName, + type: FieldType.string, + values: new ArrayVector(values), + config: {}, + }); + } + + // add the value field but clear out any labels or displayName + newFields.push({ + ...field, + name, + config: { + ...field.config, + // we need to clear thes for this transform as these can contain label names that we no longer want + displayName: undefined, + displayNameFromDS: undefined, + }, + labels: undefined, + }); } - const values = new Array(frame.length).fill(field.labels[labelName]); - newFields.push({ - name: labelName, - type: FieldType.string, - values: new ArrayVector(values), - config: {}, + result.push({ + fields: newFields, + length: frame.length, }); } - // add the value field but clear out any labels or displayName - newFields.push({ - ...field, - name, - config: { - ...field.config, - // we need to clear thes for this transform as these can contain label names that we no longer want - displayName: undefined, - displayNameFromDS: undefined, - }, - labels: undefined, - }); - } - - result.push({ - fields: newFields, - length: frame.length, - }); - } - - return mergeTransformer.transformer({})(result); - }, + return result; + }), + mergeTransformer.operator({}) + ), }; diff --git a/packages/grafana-data/src/transformations/transformers/merge.test.ts b/packages/grafana-data/src/transformations/transformers/merge.test.ts index 84ef6937de0..4f4e21716d6 100644 --- a/packages/grafana-data/src/transformations/transformers/merge.test.ts +++ b/packages/grafana-data/src/transformations/transformers/merge.test.ts @@ -1,10 +1,11 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; -import { DataTransformerConfig, Field, FieldType, DisplayProcessor } from '../../types'; +import { DataTransformerConfig, DisplayProcessor, Field, FieldType } from '../../types'; import { DataTransformerID } from './ids'; import { toDataFrame } from '../../dataframe'; import { transformDataFrame } from '../transformDataFrame'; import { ArrayVector } from '../../vector'; import { mergeTransformer, MergeTransformerOptions } from './merge'; +import { observableTester } from '../../utils/tests/observableTester'; describe('Merge multipe to single', () => { const cfg: DataTransformerConfig = { @@ -16,7 +17,7 @@ describe('Merge multipe to single', () => { mockTransformationsRegistry([mergeTransformer]); }); - it('combine two series into one', () => { + it('combine two series into one', done => { const seriesA = toDataFrame({ name: 'A', fields: [ @@ -33,16 +34,21 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [1000, 2000]), - createField('Temp', FieldType.number, [1, -1]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [1000, 2000]), + createField('Temp', FieldType.number, [1, -1]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two series with multiple values into one', () => { + it('combine two series with multiple values into one', done => { const seriesA = toDataFrame({ name: 'A', fields: [ @@ -59,16 +65,21 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine three series into one', () => { + it('combine three series into one', done => { const seriesA = toDataFrame({ name: 'A', fields: [ @@ -93,16 +104,21 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]); - const expected: Field[] = [ - createField('Time', FieldType.time, [1000, 2000, 500]), - createField('Temp', FieldType.number, [1, -1, 2]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB, seriesC]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [1000, 2000, 500]), + createField('Temp', FieldType.number, [1, -1, 2]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine one serie and two tables into one table', () => { + it('combine one serie and two tables into one table', done => { const tableA = toDataFrame({ name: 'A', fields: [ @@ -129,17 +145,22 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [tableA, seriesB, tableB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [1000, 1000, 500]), - createField('Temp', FieldType.number, [1, -1, 2]), - createField('Humidity', FieldType.number, [10, null, 5]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [tableA, seriesB, tableB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [1000, 1000, 500]), + createField('Temp', FieldType.number, [1, -1, 2]), + createField('Humidity', FieldType.number, [10, null, 5]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine one serie and two tables with ISO dates into one table', () => { + it('combine one serie and two tables with ISO dates into one table', done => { const tableA = toDataFrame({ name: 'A', fields: [ @@ -166,17 +187,22 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [tableA, seriesB, tableC]); - const expected: Field[] = [ - createField('Time', FieldType.time, ['2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z', '2019-11-01T11:10:23Z']), - createField('Temp', FieldType.number, [1, -1, 2]), - createField('Humidity', FieldType.number, [10, null, 5]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [tableA, seriesB, tableC]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, ['2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z', '2019-11-01T11:10:23Z']), + createField('Temp', FieldType.number, [1, -1, 2]), + createField('Humidity', FieldType.number, [10, null, 5]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two tables, where first is partial overlapping, into one', () => { + it('combine two tables, where first is partial overlapping, into one', done => { const tableA = toDataFrame({ name: 'A', fields: [ @@ -202,34 +228,39 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [tableA, tableB]); - const expected: Field[] = [ - createField('Country', FieldType.string, [ - 'United States', - 'United States', - 'Mexico', - 'Germany', - 'Canada', - 'Canada', - null, - ]), - createField('AgeGroup', FieldType.string, [ - '50 or over', - '35 - 49', - '0 - 17', - '35 - 49', - '35 - 49', - '25 - 34', - '18 - 24', - ]), - createField('Sum', FieldType.number, [998, 1193, 1675, 146, 166, 219, null]), - createField('Count', FieldType.number, [2, 4, 1, 4, 4, 2, 3]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [tableA, tableB]), + expect: result => { + const expected: Field[] = [ + createField('Country', FieldType.string, [ + 'United States', + 'United States', + 'Mexico', + 'Germany', + 'Canada', + 'Canada', + null, + ]), + createField('AgeGroup', FieldType.string, [ + '50 or over', + '35 - 49', + '0 - 17', + '35 - 49', + '35 - 49', + '25 - 34', + '18 - 24', + ]), + createField('Sum', FieldType.number, [998, 1193, 1675, 146, 166, 219, null]), + createField('Count', FieldType.number, [2, 4, 1, 4, 4, 2, 3]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two tables, where second is partial overlapping, into one', () => { + it('combine two tables, where second is partial overlapping, into one', done => { /** * This behavior feels wrong. I would expect the same behavior regardless of the order * of the frames. But when testing the old table panel it had this behavior so I am @@ -260,34 +291,39 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [tableA, tableB]); - const expected: Field[] = [ - createField('AgeGroup', FieldType.string, [ - '0 - 17', - '18 - 24', - '25 - 34', - '35 - 49', - '50 or over', - '35 - 49', - '35 - 49', - ]), - createField('Count', FieldType.number, [1, 3, 2, 4, 2, null, null]), - createField('Country', FieldType.string, [ - 'Mexico', - null, - 'Canada', - 'United States', - 'United States', - 'Germany', - 'Canada', - ]), - createField('Sum', FieldType.number, [1675, null, 219, 1193, 998, 146, 166]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [tableA, tableB]), + expect: result => { + const expected: Field[] = [ + createField('AgeGroup', FieldType.string, [ + '0 - 17', + '18 - 24', + '25 - 34', + '35 - 49', + '50 or over', + '35 - 49', + '35 - 49', + ]), + createField('Count', FieldType.number, [1, 3, 2, 4, 2, null, null]), + createField('Country', FieldType.string, [ + 'Mexico', + null, + 'Canada', + 'United States', + 'United States', + 'Germany', + 'Canada', + ]), + createField('Sum', FieldType.number, [1675, null, 219, 1193, 998, 146, 166]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine three tables with multiple values into one', () => { + it('combine three tables with multiple values into one', done => { const tableA = toDataFrame({ name: 'A', fields: [ @@ -315,19 +351,23 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [tableA, tableB, tableC]); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [tableA, tableB, tableC]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126, 100, 124, 149]), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3, 1, 4, 5]), + createField('Humidity', FieldType.number, [10, 14, 55, null, null, null, 22, 25, 30]), + createField('Enabled', FieldType.boolean, [null, null, null, true, false, true, null, null, null]), + ]; - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126, 100, 124, 149]), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3, 1, 4, 5]), - createField('Humidity', FieldType.number, [10, 14, 55, null, null, null, 22, 25, 30]), - createField('Enabled', FieldType.boolean, [null, null, null, true, false, true, null, null, null]), - ]; - - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where first serie fields has displayName, into one', () => { + it('combine two time series, where first serie fields has displayName, into one', done => { const serieA = toDataFrame({ name: 'A', fields: [ @@ -344,19 +384,24 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[1].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where first serie fields has display processor, into one', () => { + it('combine two time series, where first serie fields has display processor, into one', done => { const displayProcessor: DisplayProcessor = jest.fn(); const serieA = toDataFrame({ @@ -375,19 +420,24 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126], {}, displayProcessor), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126], {}, displayProcessor), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[0].display).toBe(displayProcessor); - expect(fields).toEqual(expected); + expect(fields[0].display).toBe(displayProcessor); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where first serie fields has units, into one', () => { + it('combine two time series, where first serie fields has units, into one', done => { const serieA = toDataFrame({ name: 'A', fields: [ @@ -404,19 +454,24 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3], { units: 'celsius' }), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3], { units: 'celsius' }), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[1].config).toEqual({ units: 'celsius' }); - expect(fields).toEqual(expected); + expect(fields[1].config).toEqual({ units: 'celsius' }); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where second serie fields has units, into one', () => { + it('combine two time series, where second serie fields has units, into one', done => { const serieA = toDataFrame({ name: 'A', fields: [ @@ -433,19 +488,24 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), - createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]), + createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[1].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine one regular serie with an empty serie should return the regular serie', () => { + it('combine one regular serie with an empty serie should return the regular serie', done => { const serieA = toDataFrame({ name: 'A', fields: [ @@ -459,19 +519,24 @@ describe('Merge multipe to single', () => { fields: [], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200]), - createField('Temp', FieldType.number, [1, 4, 5]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200]), + createField('Temp', FieldType.number, [1, 4, 5]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[1].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two regular series with an empty serie should return the combination of the regular series', () => { + it('combine two regular series with an empty serie should return the combination of the regular series', done => { const serieA = toDataFrame({ name: 'A', fields: [ @@ -493,20 +558,25 @@ describe('Merge multipe to single', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB, serieC]); - const expected: Field[] = [ - createField('Time', FieldType.time, [100, 150, 200]), - createField('Temp', FieldType.number, [1, 4, 5]), - createField('Humidity', FieldType.number, [6, 7, 8]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB, serieC]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 150, 200]), + createField('Temp', FieldType.number, [1, 4, 5]), + createField('Humidity', FieldType.number, [6, 7, 8]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[1].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine multiple empty series should return one empty serie', () => { + it('combine multiple empty series should return one empty serie', done => { const serieA = toDataFrame({ name: 'A', fields: [], @@ -522,12 +592,17 @@ describe('Merge multipe to single', () => { fields: [], }); - const result = transformDataFrame([cfg], [serieA, serieB, serieC]); - const expected: Field[] = []; - const fields = unwrap(result[0].fields); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB, serieC]), + expect: result => { + const expected: Field[] = []; + const fields = unwrap(result[0].fields); - expect(fields).toEqual(expected); - expect(result.length).toEqual(1); + expect(fields).toEqual(expected); + expect(result.length).toEqual(1); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/merge.ts b/packages/grafana-data/src/transformations/transformers/merge.ts index c78a7f7f80e..911cb54bf6a 100644 --- a/packages/grafana-data/src/transformations/transformers/merge.ts +++ b/packages/grafana-data/src/transformations/transformers/merge.ts @@ -1,3 +1,5 @@ +import { map } from 'rxjs/operators'; + import { DataTransformerID } from './ids'; import { DataTransformerInfo } from '../../types/transformations'; import { DataFrame, Field } from '../../types/dataFrame'; @@ -17,97 +19,98 @@ export const mergeTransformer: DataTransformerInfo = { name: 'Merge series/tables', description: 'Merges multiple series/tables into a single serie/table', defaultOptions: {}, - transformer: (options: MergeTransformerOptions) => { - return (dataFrames: DataFrame[]) => { - if (!Array.isArray(dataFrames) || dataFrames.length === 0) { - return dataFrames; - } - - const data = dataFrames.filter(frame => frame.fields.length > 0); - - if (data.length === 0) { - return [dataFrames[0]]; - } - - const fieldNames = new Set(); - const fieldIndexByName: Record> = {}; - const fieldNamesForKey: string[] = []; - const dataFrame = new MutableDataFrame(); - - for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { - const frame = data[frameIndex]; - - for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { - const field = frame.fields[fieldIndex]; - - if (!fieldNames.has(field.name)) { - dataFrame.addField(copyFieldStructure(field)); - fieldNames.add(field.name); - } - - fieldIndexByName[field.name] = fieldIndexByName[field.name] || {}; - fieldIndexByName[field.name][frameIndex] = fieldIndex; - - if (data.length - 1 !== frameIndex) { - continue; - } - - if (fieldExistsInAllFrames(fieldIndexByName, field, data)) { - fieldNamesForKey.push(field.name); - } + operator: options => source => + source.pipe( + map(dataFrames => { + if (!Array.isArray(dataFrames) || dataFrames.length === 0) { + return dataFrames; } - } - if (fieldNamesForKey.length === 0) { - return dataFrames; - } + const data = dataFrames.filter(frame => frame.fields.length > 0); - const valuesByKey: Record>> = {}; - const valuesInOrder: ValuePointer[] = []; - const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey); - const valueMapper = createValueMapper(data, fieldNames, fieldIndexByName); + if (data.length === 0) { + return [dataFrames[0]]; + } - for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { - const frame = data[frameIndex]; + const fieldNames = new Set(); + const fieldIndexByName: Record> = {}; + const fieldNamesForKey: string[] = []; + const dataFrame = new MutableDataFrame(); - for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { - const key = keyFactory(frameIndex, valueIndex); - const value = valueMapper(frameIndex, valueIndex); + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; - if (!Array.isArray(valuesByKey[key])) { - valuesByKey[key] = [value]; - valuesInOrder.push(createPointer(key, valuesByKey)); - continue; - } + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const field = frame.fields[fieldIndex]; - let valueWasMerged = false; - - valuesByKey[key] = valuesByKey[key].map(existing => { - if (!isMergable(existing, value)) { - return existing; + if (!fieldNames.has(field.name)) { + dataFrame.addField(copyFieldStructure(field)); + fieldNames.add(field.name); } - valueWasMerged = true; - return { ...existing, ...value }; - }); - if (!valueWasMerged) { - valuesByKey[key].push(value); - valuesInOrder.push(createPointer(key, valuesByKey)); + fieldIndexByName[field.name] = fieldIndexByName[field.name] || {}; + fieldIndexByName[field.name][frameIndex] = fieldIndex; + + if (data.length - 1 !== frameIndex) { + continue; + } + + if (fieldExistsInAllFrames(fieldIndexByName, field, data)) { + fieldNamesForKey.push(field.name); + } } } - } - for (const pointer of valuesInOrder) { - const value = valuesByKey[pointer.key][pointer.index]; - - if (value) { - dataFrame.add(value, false); + if (fieldNamesForKey.length === 0) { + return dataFrames; } - } - return [dataFrame]; - }; - }, + const valuesByKey: Record>> = {}; + const valuesInOrder: ValuePointer[] = []; + const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey); + const valueMapper = createValueMapper(data, fieldNames, fieldIndexByName); + + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + + for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { + const key = keyFactory(frameIndex, valueIndex); + const value = valueMapper(frameIndex, valueIndex); + + if (!Array.isArray(valuesByKey[key])) { + valuesByKey[key] = [value]; + valuesInOrder.push(createPointer(key, valuesByKey)); + continue; + } + + let valueWasMerged = false; + + valuesByKey[key] = valuesByKey[key].map(existing => { + if (!isMergable(existing, value)) { + return existing; + } + valueWasMerged = true; + return { ...existing, ...value }; + }); + + if (!valueWasMerged) { + valuesByKey[key].push(value); + valuesInOrder.push(createPointer(key, valuesByKey)); + } + } + } + + for (const pointer of valuesInOrder) { + const value = valuesByKey[pointer.key][pointer.index]; + + if (value) { + dataFrame.add(value, false); + } + } + + return [dataFrame]; + }) + ), }; const copyFieldStructure = (field: Field): Field => { diff --git a/packages/grafana-data/src/transformations/transformers/noop.ts b/packages/grafana-data/src/transformations/transformers/noop.ts index 7feb0b853d6..6ffee346a60 100644 --- a/packages/grafana-data/src/transformations/transformers/noop.ts +++ b/packages/grafana-data/src/transformations/transformers/noop.ts @@ -1,5 +1,4 @@ import { DataTransformerID } from './ids'; -import { DataFrame } from '../../types/dataFrame'; import { DataTransformerInfo } from '../../types/transformations'; export interface NoopTransformerOptions { @@ -17,7 +16,5 @@ export const noopTransformer: DataTransformerInfo = { * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: NoopTransformerOptions) => { - return (data: DataFrame[]) => data; - }, + operator: (options: NoopTransformerOptions) => source => source, }; diff --git a/packages/grafana-data/src/transformations/transformers/order.test.ts b/packages/grafana-data/src/transformations/transformers/order.test.ts index a69dec569b5..af7924b5288 100644 --- a/packages/grafana-data/src/transformations/transformers/order.test.ts +++ b/packages/grafana-data/src/transformations/transformers/order.test.ts @@ -8,6 +8,7 @@ import { } from '@grafana/data'; import { orderFieldsTransformer, OrderFieldsTransformerOptions } from './order'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { observableTester } from '../../utils/tests/observableTester'; describe('Order Transformer', () => { beforeAll(() => { @@ -23,7 +24,7 @@ describe('Order Transformer', () => { ], }); - it('should order according to config', () => { + it('should order according to config', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.order, options: { @@ -35,40 +36,45 @@ describe('Order Transformer', () => { }, }; - const ordered = transformDataFrame([cfg], [data])[0]; - - expect(ordered.fields).toEqual([ - { - config: {}, - name: 'temperature', - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - labels: undefined, - state: { - displayName: 'temperature', - }, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const ordered = data[0]; + expect(ordered.fields).toEqual([ + { + config: {}, + name: 'temperature', + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + labels: undefined, + state: { + displayName: 'temperature', + }, + }, + { + config: {}, + name: 'humidity', + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + labels: undefined, + state: { + displayName: 'humidity', + }, + }, + { + config: {}, + name: 'time', + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + labels: undefined, + state: { + displayName: 'time', + }, + }, + ]); }, - { - config: {}, - name: 'humidity', - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - labels: undefined, - state: { - displayName: 'humidity', - }, - }, - { - config: {}, - name: 'time', - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), - labels: undefined, - state: { - displayName: 'time', - }, - }, - ]); + done, + }); }); }); @@ -82,7 +88,7 @@ describe('Order Transformer', () => { ], }); - it('should append fields missing in config at the end', () => { + it('should append fields missing in config at the end', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.order, options: { @@ -94,40 +100,45 @@ describe('Order Transformer', () => { }, }; - const ordered = transformDataFrame([cfg], [data])[0]; - - expect(ordered.fields).toEqual([ - { - config: {}, - name: 'humidity', - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - labels: undefined, - state: { - displayName: 'humidity', - }, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const ordered = data[0]; + expect(ordered.fields).toEqual([ + { + config: {}, + name: 'humidity', + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + labels: undefined, + state: { + displayName: 'humidity', + }, + }, + { + config: {}, + name: 'time', + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + labels: undefined, + state: { + displayName: 'time', + }, + }, + { + config: {}, + name: 'pressure', + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + labels: undefined, + state: { + displayName: 'pressure', + }, + }, + ]); }, - { - config: {}, - name: 'time', - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), - labels: undefined, - state: { - displayName: 'time', - }, - }, - { - config: {}, - name: 'pressure', - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - labels: undefined, - state: { - displayName: 'pressure', - }, - }, - ]); + done, + }); }); }); @@ -141,7 +152,7 @@ describe('Order Transformer', () => { ], }); - it('should keep the same order as in the incoming data', () => { + it('should keep the same order as in the incoming data', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.order, options: { @@ -149,28 +160,33 @@ describe('Order Transformer', () => { }, }; - const ordered = transformDataFrame([cfg], [data])[0]; - - expect(ordered.fields).toEqual([ - { - config: {}, - name: 'time', - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const ordered = data[0]; + expect(ordered.fields).toEqual([ + { + config: {}, + name: 'time', + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + }, + { + config: {}, + name: 'pressure', + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + { + config: {}, + name: 'humidity', + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + }, + ]); }, - { - config: {}, - name: 'pressure', - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - }, - { - config: {}, - name: 'humidity', - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - }, - ]); + done, + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/order.ts b/packages/grafana-data/src/transformations/transformers/order.ts index cf44303e618..c63757891b3 100644 --- a/packages/grafana-data/src/transformations/transformers/order.ts +++ b/packages/grafana-data/src/transformations/transformers/order.ts @@ -2,6 +2,7 @@ import { DataTransformerID } from './ids'; import { DataTransformerInfo } from '../../types/transformations'; import { DataFrame, Field } from '../../types'; import { getFieldDisplayName } from '../../field/fieldState'; +import { map } from 'rxjs/operators'; export interface OrderFieldsTransformerOptions { indexByName: Record; @@ -19,20 +20,21 @@ export const orderFieldsTransformer: DataTransformerInfo { - const orderer = createFieldsOrderer(options.indexByName); + operator: options => source => + source.pipe( + map(data => { + const orderer = createFieldsOrderer(options.indexByName); - return (data: DataFrame[]) => { - if (!Array.isArray(data) || data.length === 0) { - return data; - } + if (!Array.isArray(data) || data.length === 0) { + return data; + } - return data.map(frame => ({ - ...frame, - fields: orderer(frame.fields, data, frame), - })); - }; - }, + return data.map(frame => ({ + ...frame, + fields: orderer(frame.fields, data, frame), + })); + }) + ), }; export const createOrderFieldsComparer = (indexByName: Record) => (a: string, b: string) => { diff --git a/packages/grafana-data/src/transformations/transformers/organize.test.ts b/packages/grafana-data/src/transformations/transformers/organize.test.ts index 4b406865431..0274f0be97d 100644 --- a/packages/grafana-data/src/transformations/transformers/organize.test.ts +++ b/packages/grafana-data/src/transformations/transformers/organize.test.ts @@ -8,6 +8,7 @@ import { } from '@grafana/data'; import { organizeFieldsTransformer, OrganizeFieldsTransformerOptions } from './organize'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { observableTester } from '../../utils/tests/observableTester'; describe('OrganizeFields Transformer', () => { beforeAll(() => { @@ -24,7 +25,7 @@ describe('OrganizeFields Transformer', () => { ], }); - it('should order and filter according to config', () => { + it('should order and filter according to config', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.organize, options: { @@ -42,32 +43,37 @@ describe('OrganizeFields Transformer', () => { }, }; - const organized = transformDataFrame([cfg], [data])[0]; - - expect(organized.fields).toEqual([ - { - config: {}, - labels: undefined, - name: 'temperature', - state: { - displayName: 'temperature', - }, - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const organized = data[0]; + expect(organized.fields).toEqual([ + { + config: {}, + labels: undefined, + name: 'temperature', + state: { + displayName: 'temperature', + }, + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + { + config: { + displayName: 'renamed_humidity', + }, + labels: undefined, + name: 'humidity', + state: { + displayName: 'renamed_humidity', + }, + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + }, + ]); }, - { - config: { - displayName: 'renamed_humidity', - }, - labels: undefined, - name: 'humidity', - state: { - displayName: 'renamed_humidity', - }, - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - }, - ]); + done, + }); }); }); @@ -81,7 +87,7 @@ describe('OrganizeFields Transformer', () => { ], }); - it('should append fields missing in config at the end', () => { + it('should append fields missing in config at the end', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.organize, options: { @@ -99,32 +105,37 @@ describe('OrganizeFields Transformer', () => { }, }; - const organized = transformDataFrame([cfg], [data])[0]; - - expect(organized.fields).toEqual([ - { - labels: undefined, - config: { - displayName: 'renamed_time', - }, - name: 'time', - state: { - displayName: 'renamed_time', - }, - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const organized = data[0]; + expect(organized.fields).toEqual([ + { + labels: undefined, + config: { + displayName: 'renamed_time', + }, + name: 'time', + state: { + displayName: 'renamed_time', + }, + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + }, + { + config: {}, + labels: undefined, + name: 'pressure', + state: { + displayName: 'pressure', + }, + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + ]); }, - { - config: {}, - labels: undefined, - name: 'pressure', - state: { - displayName: 'pressure', - }, - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - }, - ]); + done, + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/organize.ts b/packages/grafana-data/src/transformations/transformers/organize.ts index 990d12e171f..515cb85f791 100644 --- a/packages/grafana-data/src/transformations/transformers/organize.ts +++ b/packages/grafana-data/src/transformations/transformers/organize.ts @@ -1,9 +1,8 @@ import { DataTransformerID } from './ids'; import { DataTransformerInfo } from '../../types/transformations'; -import { OrderFieldsTransformerOptions, orderFieldsTransformer } from './order'; +import { orderFieldsTransformer, OrderFieldsTransformerOptions } from './order'; import { filterFieldsByNameTransformer } from './filterByName'; -import { DataFrame } from '../..'; -import { RenameFieldsTransformerOptions, renameFieldsTransformer } from './rename'; +import { renameFieldsTransformer, RenameFieldsTransformerOptions } from './rename'; export interface OrganizeFieldsTransformerOptions extends OrderFieldsTransformerOptions, @@ -25,15 +24,14 @@ export const organizeFieldsTransformer: DataTransformerInfo { - const rename = renameFieldsTransformer.transformer(options); - const order = orderFieldsTransformer.transformer(options); - const filter = filterFieldsByNameTransformer.transformer({ - exclude: { names: mapToExcludeArray(options.excludeByName) }, - }); - - return (data: DataFrame[]) => rename(order(filter(data))); - }, + operator: options => source => + source.pipe( + filterFieldsByNameTransformer.operator({ + exclude: { names: mapToExcludeArray(options.excludeByName) }, + }), + orderFieldsTransformer.operator(options), + renameFieldsTransformer.operator(options) + ), }; const mapToExcludeArray = (excludeByName: Record): string[] => { diff --git a/packages/grafana-data/src/transformations/transformers/reduce.test.ts b/packages/grafana-data/src/transformations/transformers/reduce.test.ts index 147931209d2..a13a3d6859d 100644 --- a/packages/grafana-data/src/transformations/transformers/reduce.test.ts +++ b/packages/grafana-data/src/transformations/transformers/reduce.test.ts @@ -6,6 +6,7 @@ import { reduceTransformer } from './reduce'; import { transformDataFrame } from '../transformDataFrame'; import { Field, FieldType } from '../../types'; import { ArrayVector } from '../../vector'; +import { observableTester } from '../../utils/tests/observableTester'; const seriesAWithSingleField = toDataFrame({ name: 'A', @@ -46,187 +47,211 @@ describe('Reducer Transformer', () => { mockTransformationsRegistry([reduceTransformer]); }); - it('reduces multiple data frames with many fields', () => { + it('reduces multiple data frames with many fields', done => { const cfg = { id: DataTransformerID.reduce, options: { reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last], }, }; - const processed = transformDataFrame([cfg], [seriesAWithMultipleFields, seriesBWithMultipleFields]); - const expected: Field[] = [ - { - name: 'Field', - type: FieldType.string, - values: new ArrayVector(['A temperature', 'A humidity', 'B temperature', 'B humidity']), - config: {}, - }, - { - name: 'First', - type: FieldType.number, - values: new ArrayVector([3, 10000.3, 1, 11000.1]), - config: {}, - }, - { - name: 'Min', - type: FieldType.number, - values: new ArrayVector([3, 10000.3, 1, 11000.1]), - config: {}, - }, - { - name: 'Max', - type: FieldType.number, - values: new ArrayVector([6, 10000.6, 7, 11000.7]), - config: {}, - }, - { - name: 'Last', - type: FieldType.number, - values: new ArrayVector([6, 10000.6, 7, 11000.7]), - config: {}, - }, - ]; - expect(processed.length).toEqual(1); - expect(processed[0].length).toEqual(4); - expect(processed[0].fields).toEqual(expected); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesAWithMultipleFields, seriesBWithMultipleFields]), + expect: processed => { + const expected: Field[] = [ + { + name: 'Field', + type: FieldType.string, + values: new ArrayVector(['A temperature', 'A humidity', 'B temperature', 'B humidity']), + config: {}, + }, + { + name: 'First', + type: FieldType.number, + values: new ArrayVector([3, 10000.3, 1, 11000.1]), + config: {}, + }, + { + name: 'Min', + type: FieldType.number, + values: new ArrayVector([3, 10000.3, 1, 11000.1]), + config: {}, + }, + { + name: 'Max', + type: FieldType.number, + values: new ArrayVector([6, 10000.6, 7, 11000.7]), + config: {}, + }, + { + name: 'Last', + type: FieldType.number, + values: new ArrayVector([6, 10000.6, 7, 11000.7]), + config: {}, + }, + ]; + + expect(processed.length).toEqual(1); + expect(processed[0].length).toEqual(4); + expect(processed[0].fields).toEqual(expected); + }, + done, + }); }); - it('reduces multiple data frames with single field', () => { + it('reduces multiple data frames with single field', done => { const cfg = { id: DataTransformerID.reduce, options: { reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last], }, }; - const processed = transformDataFrame([cfg], [seriesAWithSingleField, seriesBWithSingleField]); - const expected: Field[] = [ - { - name: 'Field', - type: FieldType.string, - values: new ArrayVector(['A temperature', 'B temperature']), - config: {}, - }, - { - name: 'First', - type: FieldType.number, - values: new ArrayVector([3, 1]), - config: {}, - }, - { - name: 'Min', - type: FieldType.number, - values: new ArrayVector([3, 1]), - config: {}, - }, - { - name: 'Max', - type: FieldType.number, - values: new ArrayVector([6, 7]), - config: {}, - }, - { - name: 'Last', - type: FieldType.number, - values: new ArrayVector([6, 7]), - config: {}, - }, - ]; - expect(processed.length).toEqual(1); - expect(processed[0].length).toEqual(2); - expect(processed[0].fields).toEqual(expected); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesAWithSingleField, seriesBWithSingleField]), + expect: processed => { + const expected: Field[] = [ + { + name: 'Field', + type: FieldType.string, + values: new ArrayVector(['A temperature', 'B temperature']), + config: {}, + }, + { + name: 'First', + type: FieldType.number, + values: new ArrayVector([3, 1]), + config: {}, + }, + { + name: 'Min', + type: FieldType.number, + values: new ArrayVector([3, 1]), + config: {}, + }, + { + name: 'Max', + type: FieldType.number, + values: new ArrayVector([6, 7]), + config: {}, + }, + { + name: 'Last', + type: FieldType.number, + values: new ArrayVector([6, 7]), + config: {}, + }, + ]; + + expect(processed.length).toEqual(1); + expect(processed[0].length).toEqual(2); + expect(processed[0].fields).toEqual(expected); + }, + done, + }); }); - it('reduces single data frame with many fields', () => { + it('reduces single data frame with many fields', done => { const cfg = { id: DataTransformerID.reduce, options: { reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last], }, }; - const processed = transformDataFrame([cfg], [seriesAWithMultipleFields]); - const expected: Field[] = [ - { - name: 'Field', - type: FieldType.string, - values: new ArrayVector(['A temperature', 'A humidity']), - config: {}, - }, - { - name: 'First', - type: FieldType.number, - values: new ArrayVector([3, 10000.3]), - config: {}, - }, - { - name: 'Min', - type: FieldType.number, - values: new ArrayVector([3, 10000.3]), - config: {}, - }, - { - name: 'Max', - type: FieldType.number, - values: new ArrayVector([6, 10000.6]), - config: {}, - }, - { - name: 'Last', - type: FieldType.number, - values: new ArrayVector([6, 10000.6]), - config: {}, - }, - ]; - expect(processed.length).toEqual(1); - expect(processed[0].length).toEqual(2); - expect(processed[0].fields).toEqual(expected); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesAWithMultipleFields]), + expect: processed => { + const expected: Field[] = [ + { + name: 'Field', + type: FieldType.string, + values: new ArrayVector(['A temperature', 'A humidity']), + config: {}, + }, + { + name: 'First', + type: FieldType.number, + values: new ArrayVector([3, 10000.3]), + config: {}, + }, + { + name: 'Min', + type: FieldType.number, + values: new ArrayVector([3, 10000.3]), + config: {}, + }, + { + name: 'Max', + type: FieldType.number, + values: new ArrayVector([6, 10000.6]), + config: {}, + }, + { + name: 'Last', + type: FieldType.number, + values: new ArrayVector([6, 10000.6]), + config: {}, + }, + ]; + + expect(processed.length).toEqual(1); + expect(processed[0].length).toEqual(2); + expect(processed[0].fields).toEqual(expected); + }, + done, + }); }); - it('reduces single data frame with single field', () => { + it('reduces single data frame with single field', done => { const cfg = { id: DataTransformerID.reduce, options: { reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last], }, }; - const processed = transformDataFrame([cfg], [seriesAWithSingleField]); - const expected: Field[] = [ - { - name: 'Field', - type: FieldType.string, - values: new ArrayVector(['A temperature']), - config: {}, - }, - { - name: 'First', - type: FieldType.number, - values: new ArrayVector([3]), - config: {}, - }, - { - name: 'Min', - type: FieldType.number, - values: new ArrayVector([3]), - config: {}, - }, - { - name: 'Max', - type: FieldType.number, - values: new ArrayVector([6]), - config: {}, - }, - { - name: 'Last', - type: FieldType.number, - values: new ArrayVector([6]), - config: {}, - }, - ]; - expect(processed.length).toEqual(1); - expect(processed[0].length).toEqual(1); - expect(processed[0].fields).toEqual(expected); + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesAWithSingleField]), + expect: processed => { + const expected: Field[] = [ + { + name: 'Field', + type: FieldType.string, + values: new ArrayVector(['A temperature']), + config: {}, + }, + { + name: 'First', + type: FieldType.number, + values: new ArrayVector([3]), + config: {}, + }, + { + name: 'Min', + type: FieldType.number, + values: new ArrayVector([3]), + config: {}, + }, + { + name: 'Max', + type: FieldType.number, + values: new ArrayVector([6]), + config: {}, + }, + { + name: 'Last', + type: FieldType.number, + values: new ArrayVector([6]), + config: {}, + }, + ]; + + expect(processed.length).toEqual(1); + expect(processed[0].length).toEqual(1); + expect(processed[0].fields).toEqual(expected); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/reduce.ts b/packages/grafana-data/src/transformations/transformers/reduce.ts index ad0d38e6ffe..ca6dcc1e771 100644 --- a/packages/grafana-data/src/transformations/transformers/reduce.ts +++ b/packages/grafana-data/src/transformations/transformers/reduce.ts @@ -1,3 +1,5 @@ +import { map } from 'rxjs/operators'; + import { DataTransformerID } from './ids'; import { DataTransformerInfo, MatcherConfig } from '../../types/transformations'; import { fieldReducers, reduceField, ReducerID } from '../fieldReducer'; @@ -28,85 +30,87 @@ export const reduceTransformer: DataTransformerInfo = * Return a modified copy of the series. If the transform is not or should not * be applied, just return the input series */ - transformer: (options: ReduceTransformerOptions) => { - const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher; - const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : []; - const reducers = calculators.map(c => c.id); + operator: options => source => + source.pipe( + map(data => { + const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher; + const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : []; + const reducers = calculators.map(c => c.id); - return (data: DataFrame[]) => { - const processed: DataFrame[] = []; + const processed: DataFrame[] = []; - for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) { - const series = data[seriesIndex]; - const values: ArrayVector[] = []; - const fields: Field[] = []; - const byId: KeyValue = {}; - - values.push(new ArrayVector()); // The name - fields.push({ - name: 'Field', - type: FieldType.string, - values: values[0], - config: {}, - }); - - for (const info of calculators) { - const vals = new ArrayVector(); - byId[info.id] = vals; - values.push(vals); + for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) { + const series = data[seriesIndex]; + const values: ArrayVector[] = []; + const fields: Field[] = []; + const byId: KeyValue = {}; + values.push(new ArrayVector()); // The name fields.push({ - name: info.name, - type: FieldType.other, // UNKNOWN until after we call the functions - values: values[values.length - 1], + name: 'Field', + type: FieldType.string, + values: values[0], config: {}, }); + + for (const info of calculators) { + const vals = new ArrayVector(); + byId[info.id] = vals; + values.push(vals); + + fields.push({ + name: info.name, + type: FieldType.other, // UNKNOWN until after we call the functions + values: values[values.length - 1], + config: {}, + }); + } + + for (let i = 0; i < series.fields.length; i++) { + const field = series.fields[i]; + + if (field.type === FieldType.time) { + continue; + } + + if (matcher(field, series, data)) { + const results = reduceField({ + field, + reducers, + }); + + // Update the name list + const fieldName = getFieldDisplayName(field, series, data); + + values[0].buffer.push(fieldName); + + for (const info of calculators) { + const v = results[info.id]; + byId[info.id].buffer.push(v); + } + } + } + + for (const f of fields) { + const t = guessFieldTypeForField(f); + + if (t) { + f.type = t; + } + } + + processed.push({ + ...series, // Same properties, different fields + fields, + length: values[0].length, + }); } - for (let i = 0; i < series.fields.length; i++) { - const field = series.fields[i]; - - if (field.type === FieldType.time) { - continue; - } - - if (matcher(field, series, data)) { - const results = reduceField({ - field, - reducers, - }); - - // Update the name list - const fieldName = getFieldDisplayName(field, series, data); - - values[0].buffer.push(fieldName); - - for (const info of calculators) { - const v = results[info.id]; - byId[info.id].buffer.push(v); - } - } - } - - for (const f of fields) { - const t = guessFieldTypeForField(f); - - if (t) { - f.type = t; - } - } - - processed.push({ - ...series, // Same properties, different fields - fields, - length: values[0].length, - }); - } - - const withoutTime = filterFieldsTransformer.transformer({ exclude: { id: FieldMatcherID.time } })(processed); - return mergeResults(withoutTime); - }; - }, + return processed; + }), + filterFieldsTransformer.operator({ exclude: { id: FieldMatcherID.time } }), + map(mergeResults) + ), }; const mergeResults = (data: DataFrame[]) => { diff --git a/packages/grafana-data/src/transformations/transformers/rename.test.ts b/packages/grafana-data/src/transformations/transformers/rename.test.ts index cc642c99a85..b4d368d1565 100644 --- a/packages/grafana-data/src/transformations/transformers/rename.test.ts +++ b/packages/grafana-data/src/transformations/transformers/rename.test.ts @@ -6,8 +6,9 @@ import { toDataFrame, transformDataFrame, } from '@grafana/data'; -import { RenameFieldsTransformerOptions, renameFieldsTransformer } from './rename'; +import { renameFieldsTransformer, RenameFieldsTransformerOptions } from './rename'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { observableTester } from '../../utils/tests/observableTester'; describe('Rename Transformer', () => { beforeAll(() => { @@ -24,7 +25,7 @@ describe('Rename Transformer', () => { ], }); - it('should rename according to config', () => { + it('should rename according to config', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.rename, options: { @@ -36,46 +37,51 @@ describe('Rename Transformer', () => { }, }; - const renamed = transformDataFrame([cfg], [data])[0]; - - expect(renamed.fields).toEqual([ - { - config: { - displayName: 'Total time', - }, - labels: undefined, - name: 'time', - state: { - displayName: 'Total time', - }, - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const renamed = data[0]; + expect(renamed.fields).toEqual([ + { + config: { + displayName: 'Total time', + }, + labels: undefined, + name: 'time', + state: { + displayName: 'Total time', + }, + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + }, + { + config: { + displayName: 'how cold is it?', + }, + labels: undefined, + name: 'temperature', + state: { + displayName: 'how cold is it?', + }, + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + { + config: { + displayName: 'Moistiness', + }, + name: 'humidity', + labels: undefined, + state: { + displayName: 'Moistiness', + }, + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + }, + ]); }, - { - config: { - displayName: 'how cold is it?', - }, - labels: undefined, - name: 'temperature', - state: { - displayName: 'how cold is it?', - }, - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - }, - { - config: { - displayName: 'Moistiness', - }, - name: 'humidity', - labels: undefined, - state: { - displayName: 'Moistiness', - }, - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - }, - ]); + done, + }); }); }); @@ -89,7 +95,7 @@ describe('Rename Transformer', () => { ], }); - it('should not rename fields missing in config', () => { + it('should not rename fields missing in config', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.rename, options: { @@ -101,44 +107,49 @@ describe('Rename Transformer', () => { }, }; - const renamed = transformDataFrame([cfg], [data])[0]; - - expect(renamed.fields).toEqual([ - { - config: { - displayName: 'ttl', - }, - name: 'time', - labels: undefined, - state: { - displayName: 'ttl', - }, - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const renamed = data[0]; + expect(renamed.fields).toEqual([ + { + config: { + displayName: 'ttl', + }, + name: 'time', + labels: undefined, + state: { + displayName: 'ttl', + }, + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + }, + { + config: {}, + labels: undefined, + name: 'pressure', + state: { + displayName: 'pressure', + }, + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + { + config: { + displayName: 'hum', + }, + labels: undefined, + name: 'humidity', + state: { + displayName: 'hum', + }, + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + }, + ]); }, - { - config: {}, - labels: undefined, - name: 'pressure', - state: { - displayName: 'pressure', - }, - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - }, - { - config: { - displayName: 'hum', - }, - labels: undefined, - name: 'humidity', - state: { - displayName: 'hum', - }, - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - }, - ]); + done, + }); }); }); @@ -152,7 +163,7 @@ describe('Rename Transformer', () => { ], }); - it('should keep the same names as in the incoming data', () => { + it('should keep the same names as in the incoming data', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.rename, options: { @@ -160,28 +171,33 @@ describe('Rename Transformer', () => { }, }; - const renamed = transformDataFrame([cfg], [data])[0]; - - expect(renamed.fields).toEqual([ - { - config: {}, - name: 'time', - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000]), + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [data]), + expect: data => { + const renamed = data[0]; + expect(renamed.fields).toEqual([ + { + config: {}, + name: 'time', + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000]), + }, + { + config: {}, + name: 'pressure', + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), + }, + { + config: {}, + name: 'humidity', + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), + }, + ]); }, - { - config: {}, - name: 'pressure', - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6]), - }, - { - config: {}, - name: 'humidity', - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]), - }, - ]); + done, + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/rename.ts b/packages/grafana-data/src/transformations/transformers/rename.ts index 47218e89d60..e4cd440bd90 100644 --- a/packages/grafana-data/src/transformations/transformers/rename.ts +++ b/packages/grafana-data/src/transformations/transformers/rename.ts @@ -2,6 +2,7 @@ import { DataTransformerID } from './ids'; import { DataTransformerInfo } from '../../types/transformations'; import { DataFrame, Field } from '../../types/dataFrame'; import { getFieldDisplayName } from '../../field/fieldState'; +import { map } from 'rxjs/operators'; export interface RenameFieldsTransformerOptions { renameByName: Record; @@ -19,20 +20,21 @@ export const renameFieldsTransformer: DataTransformerInfo { - const renamer = createRenamer(options.renameByName); + operator: options => source => + source.pipe( + map(data => { + const renamer = createRenamer(options.renameByName); - return (data: DataFrame[]) => { - if (!Array.isArray(data) || data.length === 0) { - return data; - } + if (!Array.isArray(data) || data.length === 0) { + return data; + } - return data.map(frame => ({ - ...frame, - fields: renamer(frame), - })); - }; - }, + return data.map(frame => ({ + ...frame, + fields: renamer(frame), + })); + }) + ), }; const createRenamer = (renameByName: Record) => (frame: DataFrame): Field[] => { diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts index b7082341eba..c48a14dfe87 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts @@ -9,6 +9,7 @@ import { } from '@grafana/data'; import { SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { observableTester } from '../../utils/tests/observableTester'; describe('SeriesToColumns Transformer', () => { beforeAll(() => { @@ -33,7 +34,7 @@ describe('SeriesToColumns Transformer', () => { ], }); - it('joins by time field', () => { + it('joins by time field', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -41,62 +42,68 @@ describe('SeriesToColumns Transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; - expect(filtered.fields).toEqual([ - { - name: 'time', - state: { - displayName: 'time', - }, - type: FieldType.time, - values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), - config: {}, - labels: undefined, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + state: { + displayName: 'time', + }, + type: FieldType.time, + values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), + config: {}, + labels: undefined, + }, + { + name: 'temperature', + state: { + displayName: 'temperature even', + }, + type: FieldType.number, + values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity even', + }, + type: FieldType.number, + values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'temperature', + state: { + displayName: 'temperature odd', + }, + type: FieldType.number, + values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), + config: {}, + labels: { name: 'odd' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity odd', + }, + type: FieldType.number, + values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), + config: {}, + labels: { name: 'odd' }, + }, + ]); }, - { - name: 'temperature', - state: { - displayName: 'temperature even', - }, - type: FieldType.number, - values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity even', - }, - type: FieldType.number, - values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'temperature', - state: { - displayName: 'temperature odd', - }, - type: FieldType.number, - values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), - config: {}, - labels: { name: 'odd' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity odd', - }, - type: FieldType.number, - values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), - config: {}, - labels: { name: 'odd' }, - }, - ]); + done, + }); }); - it('joins by temperature field', () => { + it('joins by temperature field', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -104,62 +111,68 @@ describe('SeriesToColumns Transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; - expect(filtered.fields).toEqual([ - { - name: 'temperature', - state: { - displayName: 'temperature', - }, - type: FieldType.number, - values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]), - config: {}, - labels: undefined, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields).toEqual([ + { + name: 'temperature', + state: { + displayName: 'temperature', + }, + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]), + config: {}, + labels: undefined, + }, + { + name: 'time', + state: { + displayName: 'time even', + }, + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity even', + }, + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'time', + state: { + displayName: 'time odd', + }, + type: FieldType.time, + values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]), + config: {}, + labels: { name: 'odd' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity odd', + }, + type: FieldType.number, + values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]), + config: {}, + labels: { name: 'odd' }, + }, + ]); }, - { - name: 'time', - state: { - displayName: 'time even', - }, - type: FieldType.time, - values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity even', - }, - type: FieldType.number, - values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'time', - state: { - displayName: 'time odd', - }, - type: FieldType.time, - values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]), - config: {}, - labels: { name: 'odd' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity odd', - }, - type: FieldType.number, - values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]), - config: {}, - labels: { name: 'odd' }, - }, - ]); + done, + }); }); - it('joins by time field in reverse order', () => { + it('joins by time field in reverse order', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -171,59 +184,65 @@ describe('SeriesToColumns Transformer', () => { everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse()); everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse()); - const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; - expect(filtered.fields).toEqual([ - { - name: 'time', - state: { - displayName: 'time', - }, - type: FieldType.time, - values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), - config: {}, - labels: undefined, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + state: { + displayName: 'time', + }, + type: FieldType.time, + values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), + config: {}, + labels: undefined, + }, + { + name: 'temperature', + state: { + displayName: 'temperature even', + }, + type: FieldType.number, + values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity even', + }, + type: FieldType.number, + values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), + config: {}, + labels: { name: 'even' }, + }, + { + name: 'temperature', + state: { + displayName: 'temperature odd', + }, + type: FieldType.number, + values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), + config: {}, + labels: { name: 'odd' }, + }, + { + name: 'humidity', + state: { + displayName: 'humidity odd', + }, + type: FieldType.number, + values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), + config: {}, + labels: { name: 'odd' }, + }, + ]); }, - { - name: 'temperature', - state: { - displayName: 'temperature even', - }, - type: FieldType.number, - values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity even', - }, - type: FieldType.number, - values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), - config: {}, - labels: { name: 'even' }, - }, - { - name: 'temperature', - state: { - displayName: 'temperature odd', - }, - type: FieldType.number, - values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), - config: {}, - labels: { name: 'odd' }, - }, - { - name: 'humidity', - state: { - displayName: 'humidity odd', - }, - type: FieldType.number, - values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), - config: {}, - labels: { name: 'odd' }, - }, - ]); + done, + }); }); describe('Field names', () => { @@ -243,7 +262,7 @@ describe('SeriesToColumns Transformer', () => { ], }); - it('when dataframe and field share the same name then use the field name', () => { + it('when dataframe and field share the same name then use the field name', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -251,45 +270,51 @@ describe('SeriesToColumns Transformer', () => { }, }; - const filtered = transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB])[0]; - const expected: Field[] = [ - { - name: 'time', - state: { - displayName: 'time', - }, - type: FieldType.time, - values: new ArrayVector([1000, 2000, 3000, 4000]), - config: {}, - labels: undefined, - }, - { - name: 'temperature', - type: FieldType.number, - values: new ArrayVector([1, 3, 5, 7]), - config: {}, - state: { - displayName: 'temperature temperature', - }, - labels: { name: 'temperature' }, - }, - { - name: 'temperature', - state: { - displayName: 'temperature B', - }, - type: FieldType.number, - values: new ArrayVector([2, 4, 6, 8]), - config: {}, - labels: { name: 'B' }, - }, - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB]), + expect: data => { + const filtered = data[0]; + const expected: Field[] = [ + { + name: 'time', + state: { + displayName: 'time', + }, + type: FieldType.time, + values: new ArrayVector([1000, 2000, 3000, 4000]), + config: {}, + labels: undefined, + }, + { + name: 'temperature', + type: FieldType.number, + values: new ArrayVector([1, 3, 5, 7]), + config: {}, + state: { + displayName: 'temperature temperature', + }, + labels: { name: 'temperature' }, + }, + { + name: 'temperature', + state: { + displayName: 'temperature B', + }, + type: FieldType.number, + values: new ArrayVector([2, 4, 6, 8]), + config: {}, + labels: { name: 'B' }, + }, + ]; - expect(filtered.fields).toEqual(expected); + expect(filtered.fields).toEqual(expected); + }, + done, + }); }); }); - it('joins if fields are missing', () => { + it('joins if fields are missing', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -318,36 +343,41 @@ describe('SeriesToColumns Transformer', () => { ], }); - const filtered = transformDataFrame([cfg], [frame1, frame2, frame3])[0]; - - expect(filtered.fields).toEqual([ - { - name: 'time', - state: { displayName: 'time' }, - type: FieldType.time, - values: new ArrayVector([1, 2, 3]), - config: {}, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [frame1, frame2, frame3]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + state: { displayName: 'time' }, + type: FieldType.time, + values: new ArrayVector([1, 2, 3]), + config: {}, + }, + { + name: 'temperature', + state: { displayName: 'temperature A' }, + type: FieldType.number, + values: new ArrayVector([10, 11, 12]), + config: {}, + labels: { name: 'A' }, + }, + { + name: 'temperature', + state: { displayName: 'temperature C' }, + type: FieldType.number, + values: new ArrayVector([20, 22, 24]), + config: {}, + labels: { name: 'C' }, + }, + ]); }, - { - name: 'temperature', - state: { displayName: 'temperature A' }, - type: FieldType.number, - values: new ArrayVector([10, 11, 12]), - config: {}, - labels: { name: 'A' }, - }, - { - name: 'temperature', - state: { displayName: 'temperature C' }, - type: FieldType.number, - values: new ArrayVector([20, 22, 24]), - config: {}, - labels: { name: 'C' }, - }, - ]); + done, + }); }); - it('handles duplicate field name', () => { + it('handles duplicate field name', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { @@ -369,32 +399,37 @@ describe('SeriesToColumns Transformer', () => { ], }); - const filtered = transformDataFrame([cfg], [frame1, frame2])[0]; - - expect(filtered.fields).toEqual([ - { - name: 'time', - state: { displayName: 'time' }, - type: FieldType.time, - values: new ArrayVector([1]), - config: {}, + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [frame1, frame2]), + expect: data => { + const filtered = data[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + state: { displayName: 'time' }, + type: FieldType.time, + values: new ArrayVector([1]), + config: {}, + }, + { + name: 'temperature', + state: { displayName: 'temperature 1' }, + type: FieldType.number, + values: new ArrayVector([10]), + config: {}, + labels: {}, + }, + { + name: 'temperature', + state: { displayName: 'temperature 2' }, + type: FieldType.number, + values: new ArrayVector([20]), + config: {}, + labels: {}, + }, + ]); }, - { - name: 'temperature', - state: { displayName: 'temperature 1' }, - type: FieldType.number, - values: new ArrayVector([10]), - config: {}, - labels: {}, - }, - { - name: 'temperature', - state: { displayName: 'temperature 2' }, - type: FieldType.number, - values: new ArrayVector([20]), - config: {}, - labels: {}, - }, - ]); + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts index 3b8eea79b87..aed6ddb0ae3 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts @@ -1,3 +1,5 @@ +import { map } from 'rxjs/operators'; + import { DataFrame, DataTransformerInfo, Field } from '../../types'; import { DataTransformerID } from './ids'; import { MutableDataFrame } from '../../dataframe'; @@ -17,64 +19,66 @@ export const seriesToColumnsTransformer: DataTransformerInfo (data: DataFrame[]) => { - const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD; - const allFields: FieldsToProcess[] = []; + operator: options => source => + source.pipe( + map(data => { + const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD; + const allFields: FieldsToProcess[] = []; - for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { - const frame = data[frameIndex]; - const keyField = findKeyField(frame, keyFieldMatch); + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + const keyField = findKeyField(frame, keyFieldMatch); - if (!keyField) { - continue; - } + if (!keyField) { + continue; + } - for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { - const sourceField = frame.fields[fieldIndex]; + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const sourceField = frame.fields[fieldIndex]; - if (sourceField === keyField) { - continue; + if (sourceField === keyField) { + continue; + } + + let labels = sourceField.labels ?? {}; + + if (frame.name) { + labels = { ...labels, name: frame.name }; + } + + allFields.push({ + keyField, + sourceField, + newField: { + ...sourceField, + state: null, + values: new ArrayVector([]), + labels, + }, + }); + } } - let labels = sourceField.labels ?? {}; - - if (frame.name) { - labels = { ...labels, name: frame.name }; + // if no key fields or more than one value field + if (allFields.length <= 1) { + return data; } - allFields.push({ - keyField, - sourceField, - newField: { - ...sourceField, - state: null, - values: new ArrayVector([]), - labels, - }, + const resultFrame = new MutableDataFrame(); + + resultFrame.addField({ + ...allFields[0].keyField, + values: new ArrayVector([]), }); - } - } - // if no key fields or more than one value field - if (allFields.length <= 1) { - return data; - } + for (const item of allFields) { + item.newField = resultFrame.addField(item.newField); + } - const resultFrame = new MutableDataFrame(); + const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame); + const byKeyField: { [key: string]: { [key: string]: any } } = {}; - resultFrame.addField({ - ...allFields[0].keyField, - values: new ArrayVector([]), - }); - - for (const item of allFields) { - item.newField = resultFrame.addField(item.newField); - } - - const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame); - const byKeyField: { [key: string]: { [key: string]: any } } = {}; - - /* + /* this loop creates a dictionary object that groups the key fields values { "key field first value as string" : { @@ -90,36 +94,37 @@ export const seriesToColumnsTransformer: DataTransformerInfo { beforeAll(() => { mockTransformationsRegistry([seriesToRowsTransformer]); }); - it('combine two series into one', () => { + it('combine two series into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -33,17 +34,22 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [2000, 1000]), - createField('Metric', FieldType.string, ['B', 'A']), - createField('Value', FieldType.number, [-1, 1]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [2000, 1000]), + createField('Metric', FieldType.string, ['B', 'A']), + createField('Value', FieldType.number, [-1, 1]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two series with multiple values into one', () => { + it('combine two series with multiple values into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -65,17 +71,22 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), - createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), - createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine three series into one', () => { + it('combine three series into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -105,17 +116,22 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]); - const expected: Field[] = [ - createField('Time', FieldType.time, [2000, 1000, 500]), - createField('Metric', FieldType.string, ['B', 'A', 'C']), - createField('Value', FieldType.number, [-1, 1, 2]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [seriesA, seriesB, seriesC]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [2000, 1000, 500]), + createField('Metric', FieldType.string, ['B', 'A', 'C']), + createField('Value', FieldType.number, [-1, 1, 2]), + ]; - expect(unwrap(result[0].fields)).toEqual(expected); + expect(unwrap(result[0].fields)).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where first serie fields has displayName, into one', () => { + it('combine two time series, where first serie fields has displayName, into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -137,20 +153,25 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), - createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), - createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[2].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[2].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where first serie fields has units, into one', () => { + it('combine two time series, where first serie fields has units, into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -172,20 +193,25 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), - createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), - createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[2].config).toEqual({ units: 'celsius' }); - expect(fields).toEqual(expected); + expect(fields[2].config).toEqual({ units: 'celsius' }); + expect(fields).toEqual(expected); + }, + done, + }); }); - it('combine two time series, where second serie fields has units, into one', () => { + it('combine two time series, where second serie fields has units, into one', done => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToRows, options: {}, @@ -207,17 +233,22 @@ describe('Series to rows', () => { ], }); - const result = transformDataFrame([cfg], [serieA, serieB]); - const expected: Field[] = [ - createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), - createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), - createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), - ]; + observableTester().subscribeAndExpectOnNext({ + observable: transformDataFrame([cfg], [serieA, serieB]), + expect: result => { + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; - const fields = unwrap(result[0].fields); + const fields = unwrap(result[0].fields); - expect(fields[2].config).toEqual({}); - expect(fields).toEqual(expected); + expect(fields[2].config).toEqual({}); + expect(fields).toEqual(expected); + }, + done, + }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/seriesToRows.ts b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts index ee2966a7ed5..ae316891689 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToRows.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts @@ -1,13 +1,14 @@ import { omit } from 'lodash'; +import { map } from 'rxjs/operators'; + import { DataTransformerID } from './ids'; import { DataTransformerInfo } from '../../types/transformations'; import { - DataFrame, Field, FieldType, + TIME_SERIES_METRIC_FIELD_NAME, TIME_SERIES_TIME_FIELD_NAME, TIME_SERIES_VALUE_FIELD_NAME, - TIME_SERIES_METRIC_FIELD_NAME, } from '../../types/dataFrame'; import { isTimeSeries } from '../../dataframe/utils'; import { MutableDataFrame, sortDataFrame } from '../../dataframe'; @@ -21,68 +22,69 @@ export const seriesToRowsTransformer: DataTransformerInfo { - return (data: DataFrame[]) => { - if (!Array.isArray(data) || data.length <= 1) { - return data; - } + operator: options => source => + source.pipe( + map(data => { + if (!Array.isArray(data) || data.length <= 1) { + return data; + } - if (!isTimeSeries(data)) { - return data; - } + if (!isTimeSeries(data)) { + return data; + } - const timeFieldByIndex: Record = {}; - const targetFields = new Set(); - const dataFrame = new MutableDataFrame(); - const metricField: Field = { - name: TIME_SERIES_METRIC_FIELD_NAME, - values: new ArrayVector(), - config: {}, - type: FieldType.string, - }; + const timeFieldByIndex: Record = {}; + const targetFields = new Set(); + const dataFrame = new MutableDataFrame(); + const metricField: Field = { + name: TIME_SERIES_METRIC_FIELD_NAME, + values: new ArrayVector(), + config: {}, + type: FieldType.string, + }; - for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { - const frame = data[frameIndex]; + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; - for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { - const field = frame.fields[fieldIndex]; + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const field = frame.fields[fieldIndex]; - if (field.type === FieldType.time) { - timeFieldByIndex[frameIndex] = fieldIndex; + if (field.type === FieldType.time) { + timeFieldByIndex[frameIndex] = fieldIndex; - if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) { - dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME)); - dataFrame.addField(metricField); - targetFields.add(TIME_SERIES_TIME_FIELD_NAME); + if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) { + dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME)); + dataFrame.addField(metricField); + targetFields.add(TIME_SERIES_TIME_FIELD_NAME); + } + continue; } - continue; - } - if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) { - dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME)); - targetFields.add(TIME_SERIES_VALUE_FIELD_NAME); + if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) { + dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME)); + targetFields.add(TIME_SERIES_VALUE_FIELD_NAME); + } } } - } - for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { - const frame = data[frameIndex]; + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; - for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { - const timeFieldIndex = timeFieldByIndex[frameIndex]; - const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0; + for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { + const timeFieldIndex = timeFieldByIndex[frameIndex]; + const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0; - dataFrame.add({ - [TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex), - [TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame), - [TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex), - }); + dataFrame.add({ + [TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex), + [TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame), + [TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex), + }); + } } - } - return [sortDataFrame(dataFrame, 0, true)]; - }; - }, + return [sortDataFrame(dataFrame, 0, true)]; + }) + ), }; const copyFieldStructure = (field: Field, name: string): Field => { diff --git a/packages/grafana-data/src/types/annotations.ts b/packages/grafana-data/src/types/annotations.ts index 49916ce109a..c2858bd1b83 100644 --- a/packages/grafana-data/src/types/annotations.ts +++ b/packages/grafana-data/src/types/annotations.ts @@ -1,6 +1,8 @@ +import { Observable } from 'rxjs'; +import { ComponentType } from 'react'; + import { DataQuery, QueryEditorProps } from './datasource'; import { DataFrame } from './dataFrame'; -import { ComponentType } from 'react'; /** * This JSON object is stored in the dashboard json model. @@ -79,7 +81,7 @@ export interface AnnotationSupport event processing is insufficient, this allows explicit control of the mappings */ - processEvents?(anno: TAnno, data: DataFrame[]): AnnotationEvent[] | undefined; + processEvents?(anno: TAnno, data: DataFrame[]): Observable; /** * Specify a custom QueryEditor for the annotation page. If not specified, the standard one will be used diff --git a/packages/grafana-data/src/types/transformations.ts b/packages/grafana-data/src/types/transformations.ts index 98b379d2880..386ae810619 100644 --- a/packages/grafana-data/src/types/transformations.ts +++ b/packages/grafana-data/src/types/transformations.ts @@ -1,17 +1,17 @@ +import { MonoTypeOperatorFunction } from 'rxjs'; + import { DataFrame, Field } from './dataFrame'; import { RegistryItemWithOptions } from '../utils/Registry'; /** * Function that transform data frames (AKA transformer) */ -export type DataTransformer = (data: DataFrame[]) => DataFrame[]; - export interface DataTransformerInfo extends RegistryItemWithOptions { /** * Function that configures transformation and returns a transformer * @param options */ - transformer: (options: TOptions) => DataTransformer; + operator: (options: TOptions) => MonoTypeOperatorFunction; } export interface DataTransformerConfig { diff --git a/packages/grafana-data/src/utils/index.ts b/packages/grafana-data/src/utils/index.ts index 0f3e90bf2b7..d142b1c9ab4 100644 --- a/packages/grafana-data/src/utils/index.ts +++ b/packages/grafana-data/src/utils/index.ts @@ -17,3 +17,4 @@ export { locationUtil } from './location'; export { urlUtil, UrlQueryMap, UrlQueryValue } from './url'; export { DataLinkBuiltInVars } from './dataLinks'; export { DocsId } from './docs'; +export { observableTester } from './tests/observableTester'; diff --git a/public/test/helpers/observableTester.ts b/packages/grafana-data/src/utils/tests/observableTester.ts similarity index 96% rename from public/test/helpers/observableTester.ts rename to packages/grafana-data/src/utils/tests/observableTester.ts index f0a80047b6b..b5ba5e488a8 100644 --- a/public/test/helpers/observableTester.ts +++ b/packages/grafana-data/src/utils/tests/observableTester.ts @@ -2,7 +2,10 @@ import { Observable } from 'rxjs'; interface ObservableTester { observable: Observable; - done: jest.DoneCallback; + done: { + (...args: any[]): any; + fail(error?: string | { message: string }): any; + }; } interface SubscribeAndExpectOnNext extends ObservableTester { diff --git a/public/app/core/components/TransformersUI/CalculateFieldTransformerEditor.tsx b/public/app/core/components/TransformersUI/CalculateFieldTransformerEditor.tsx index 41f90e873f3..822482e07eb 100644 --- a/public/app/core/components/TransformersUI/CalculateFieldTransformerEditor.tsx +++ b/public/app/core/components/TransformersUI/CalculateFieldTransformerEditor.tsx @@ -1,25 +1,27 @@ import React, { ChangeEvent } from 'react'; - +import { of, OperatorFunction } from 'rxjs'; +import { map } from 'rxjs/operators'; import { + BinaryOperationID, + binaryOperators, + DataFrame, DataTransformerID, FieldType, + getFieldDisplayName, KeyValue, ReducerID, + SelectableValue, standardTransformers, TransformerRegistyItem, TransformerUIProps, - BinaryOperationID, - SelectableValue, - binaryOperators, - getFieldDisplayName, } from '@grafana/data'; -import { Select, StatsPicker, LegacyForms, Input, FilterPill, HorizontalGroup } from '@grafana/ui'; +import { FilterPill, HorizontalGroup, Input, LegacyForms, Select, StatsPicker } from '@grafana/ui'; import { - CalculateFieldTransformerOptions, + BinaryOptions, CalculateFieldMode, + CalculateFieldTransformerOptions, getNameFromOptions, ReduceOptions, - BinaryOptions, } from '@grafana/data/src/transformations/transformers/calculateField'; import defaults from 'lodash/defaults'; @@ -64,44 +66,67 @@ export class CalculateFieldTransformerEditor extends React.PureComponent< private initOptions() { const { options } = this.props; const configuredOptions = options?.reduce?.include || []; - const input = standardTransformers.ensureColumnsTransformer.transformer(null)(this.props.input); - - const allNames: string[] = []; - const byName: KeyValue = {}; - - for (const frame of input) { - for (const field of frame.fields) { - if (field.type !== FieldType.number) { - continue; - } - - const displayName = getFieldDisplayName(field, frame, input); - - if (!byName[displayName]) { - byName[displayName] = true; - allNames.push(displayName); - } - } - } - - if (configuredOptions.length) { - const options: string[] = []; - const selected: string[] = []; - - for (const v of allNames) { - if (configuredOptions.includes(v)) { - selected.push(v); - } - options.push(v); - } - - this.setState({ - names: options, - selected: selected, + const subscription = of(this.props.input) + .pipe( + standardTransformers.ensureColumnsTransformer.operator(null), + this.extractAllNames(), + this.extractNamesAndSelected(configuredOptions) + ) + .subscribe(({ selected, names }) => { + this.setState({ names, selected }, () => subscription.unsubscribe()); }); - } else { - this.setState({ names: allNames, selected: [] }); - } + } + + private extractAllNames(): OperatorFunction { + return source => + source.pipe( + map(input => { + const allNames: string[] = []; + const byName: KeyValue = {}; + + for (const frame of input) { + for (const field of frame.fields) { + if (field.type !== FieldType.number) { + continue; + } + + const displayName = getFieldDisplayName(field, frame, input); + + if (!byName[displayName]) { + byName[displayName] = true; + allNames.push(displayName); + } + } + } + + return allNames; + }) + ); + } + + private extractNamesAndSelected( + configuredOptions: string[] + ): OperatorFunction { + return source => + source.pipe( + map(allNames => { + if (!configuredOptions.length) { + return { names: allNames, selected: [] }; + } + + const names: string[] = []; + const selected: string[] = []; + + for (const v of allNames) { + if (configuredOptions.includes(v)) { + selected.push(v); + } + names.push(v); + } + + return { names, selected }; + }) + ); } onToggleReplaceFields = () => { diff --git a/public/app/features/annotations/annotations_srv.ts b/public/app/features/annotations/annotations_srv.ts index 4a0a9e262c5..6540a5aca93 100644 --- a/public/app/features/annotations/annotations_srv.ts +++ b/public/app/features/annotations/annotations_srv.ts @@ -11,19 +11,19 @@ import { DashboardModel } from '../dashboard/state'; import { AnnotationEvent, AppEvents, + CoreApp, + DataQueryRequest, DataSourceApi, PanelEvents, rangeUtil, - DataQueryRequest, - CoreApp, ScopedVars, } from '@grafana/data'; import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime'; import { appEvents } from 'app/core/core'; import { getTimeSrv } from '../dashboard/services/TimeSrv'; import { Observable, of } from 'rxjs'; -import { map } from 'rxjs/operators'; -import { AnnotationQueryResponse, AnnotationQueryOptions } from './types'; +import { map, mergeMap } from 'rxjs/operators'; +import { AnnotationQueryOptions, AnnotationQueryResponse } from './types'; import { standardAnnotationSupport } from './standardAnnotationSupport'; import { runRequest } from '../dashboard/state/runRequest'; @@ -264,9 +264,12 @@ export function executeAnnotationQuery( }; return runRequest(datasource, queryRequest).pipe( - map(panelData => { - const events = panelData.series ? processor.processEvents!(annotation, panelData.series) : []; - return { panelData, events }; + mergeMap(panelData => { + if (!panelData.series) { + return of({ panelData, events: [] }); + } + + return processor.processEvents!(annotation, panelData.series).pipe(map(events => ({ panelData, events }))); }) ); } diff --git a/public/app/features/annotations/standardAnnotationSupport.test.ts b/public/app/features/annotations/standardAnnotationSupport.test.ts index 092202c9f94..6a50f5ceadc 100644 --- a/public/app/features/annotations/standardAnnotationSupport.test.ts +++ b/public/app/features/annotations/standardAnnotationSupport.test.ts @@ -1,8 +1,8 @@ -import { toDataFrame, FieldType } from '@grafana/data'; +import { FieldType, observableTester, toDataFrame } from '@grafana/data'; import { getAnnotationsFromData } from './standardAnnotationSupport'; describe('DataFrame to annotations', () => { - test('simple conversion', () => { + test('simple conversion', done => { const frame = toDataFrame({ fields: [ { type: FieldType.time, values: [1, 2, 3, 4, 5] }, @@ -11,53 +11,48 @@ describe('DataFrame to annotations', () => { ], }); - const events = getAnnotationsFromData([frame]); - expect(events).toMatchInlineSnapshot(` - Array [ - Object { - "color": "red", - "tags": Array [ - "aaa", - "bbb", - ], - "text": "t1", - "time": 1, - "type": "default", - }, - Object { - "color": "red", - "tags": Array [ - "bbb", - "ccc", - ], - "text": "t2", - "time": 2, - "type": "default", - }, - Object { - "color": "red", - "tags": Array [ - "zyz", - ], - "text": "t3", - "time": 3, - "type": "default", - }, - Object { - "color": "red", - "time": 4, - "type": "default", - }, - Object { - "color": "red", - "time": 5, - "type": "default", - }, - ] - `); + observableTester().subscribeAndExpectOnNext({ + observable: getAnnotationsFromData([frame]), + expect: events => { + expect(events).toEqual([ + { + color: 'red', + tags: ['aaa', 'bbb'], + text: 't1', + time: 1, + type: 'default', + }, + { + color: 'red', + tags: ['bbb', 'ccc'], + text: 't2', + time: 2, + type: 'default', + }, + { + color: 'red', + tags: ['zyz'], + text: 't3', + time: 3, + type: 'default', + }, + { + color: 'red', + time: 4, + type: 'default', + }, + { + color: 'red', + time: 5, + type: 'default', + }, + ]); + }, + done, + }); }); - test('explicit mappins', () => { + test('explicit mappins', done => { const frame = toDataFrame({ fields: [ { name: 'time1', values: [111, 222, 333] }, @@ -67,40 +62,42 @@ describe('DataFrame to annotations', () => { ], }); - const events = getAnnotationsFromData([frame], { - text: { value: 'bbbbb' }, - time: { value: 'time2' }, - timeEnd: { value: 'time1' }, - title: { value: 'aaaaa' }, + observableTester().subscribeAndExpectOnNext({ + observable: getAnnotationsFromData([frame], { + text: { value: 'bbbbb' }, + time: { value: 'time2' }, + timeEnd: { value: 'time1' }, + title: { value: 'aaaaa' }, + }), + expect: events => { + expect(events).toEqual([ + { + color: 'red', + text: 'b1', + time: 100, + timeEnd: 111, + title: 'a1', + type: 'default', + }, + { + color: 'red', + text: 'b2', + time: 200, + timeEnd: 222, + title: 'a2', + type: 'default', + }, + { + color: 'red', + text: 'b3', + time: 300, + timeEnd: 333, + title: 'a3', + type: 'default', + }, + ]); + }, + done, }); - - expect(events).toMatchInlineSnapshot(` - Array [ - Object { - "color": "red", - "text": "b1", - "time": 100, - "timeEnd": 111, - "title": "a1", - "type": "default", - }, - Object { - "color": "red", - "text": "b2", - "time": 200, - "timeEnd": 222, - "title": "a2", - "type": "default", - }, - Object { - "color": "red", - "text": "b3", - "time": 300, - "timeEnd": 333, - "title": "a3", - "type": "default", - }, - ] - `); }); }); diff --git a/public/app/features/annotations/standardAnnotationSupport.ts b/public/app/features/annotations/standardAnnotationSupport.ts index 449e79085dc..e9197f4d41d 100644 --- a/public/app/features/annotations/standardAnnotationSupport.ts +++ b/public/app/features/annotations/standardAnnotationSupport.ts @@ -1,15 +1,17 @@ +import { Observable, of, OperatorFunction } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; import { - DataFrame, + AnnotationEvent, + AnnotationEventFieldSource, + AnnotationEventMappings, AnnotationQuery, AnnotationSupport, - standardTransformers, - FieldType, + DataFrame, Field, - KeyValue, - AnnotationEvent, - AnnotationEventMappings, + FieldType, getFieldDisplayName, - AnnotationEventFieldSource, + KeyValue, + standardTransformers, } from '@grafana/data'; import isString from 'lodash/isString'; @@ -50,16 +52,25 @@ export const standardAnnotationSupport: AnnotationSupport = { /** * Flatten all panel data into a single frame */ -export function singleFrameFromPanelData(data: DataFrame[]): DataFrame | undefined { - if (!data?.length) { - return undefined; - } - if (data.length === 1) { - return data[0]; - } +export function singleFrameFromPanelData(): OperatorFunction { + return source => + source.pipe( + mergeMap(data => { + if (!data?.length) { + return of(undefined); + } - return standardTransformers.mergeTransformer.transformer({})(data)[0]; + if (data.length === 1) { + return of(data[0]); + } + + return of(data).pipe( + standardTransformers.mergeTransformer.operator({}), + map(d => d[0]) + ); + }) + ); } interface AnnotationEventFieldSetter { @@ -100,96 +111,102 @@ export const annotationEventNames: AnnotationFieldInfo[] = [ // { key: 'email' }, ]; -export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEventMappings): AnnotationEvent[] { - const frame = singleFrameFromPanelData(data); - - if (!frame?.length) { - return []; - } - - let hasTime = false; - let hasText = false; - const byName: KeyValue = {}; - - for (const f of frame.fields) { - const name = getFieldDisplayName(f, frame); - byName[name.toLowerCase()] = f; - } - - if (!options) { - options = {}; - } - - const fields: AnnotationEventFieldSetter[] = []; - - for (const evts of annotationEventNames) { - const opt = options[evts.key] || {}; //AnnotationEventFieldMapping - - if (opt.source === AnnotationEventFieldSource.Skip) { - continue; - } - - const setter: AnnotationEventFieldSetter = { key: evts.key, split: evts.split }; - - if (opt.source === AnnotationEventFieldSource.Text) { - setter.text = opt.value; - } else { - const lower = (opt.value || evts.key).toLowerCase(); - setter.field = byName[lower]; - - if (!setter.field && evts.field) { - setter.field = evts.field(frame); +export function getAnnotationsFromData( + data: DataFrame[], + options?: AnnotationEventMappings +): Observable { + return of(data).pipe( + singleFrameFromPanelData(), + map(frame => { + if (!frame?.length) { + return []; } - } - if (setter.field || setter.text) { - fields.push(setter); - if (setter.key === 'time') { - hasTime = true; - } else if (setter.key === 'text') { - hasText = true; + let hasTime = false; + let hasText = false; + const byName: KeyValue = {}; + + for (const f of frame.fields) { + const name = getFieldDisplayName(f, frame); + byName[name.toLowerCase()] = f; } - } - } - if (!hasTime || !hasText) { - return []; // throw an error? - } + if (!options) { + options = {}; + } - // Add each value to the string - const events: AnnotationEvent[] = []; + const fields: AnnotationEventFieldSetter[] = []; - for (let i = 0; i < frame.length; i++) { - const anno: AnnotationEvent = { - type: 'default', - color: 'red', - }; + for (const evts of annotationEventNames) { + const opt = options[evts.key] || {}; //AnnotationEventFieldMapping - for (const f of fields) { - let v: any = undefined; + if (opt.source === AnnotationEventFieldSource.Skip) { + continue; + } - if (f.text) { - v = f.text; // TODO support templates! - } else if (f.field) { - v = f.field.values.get(i); - if (v !== undefined && f.regex) { - const match = f.regex.exec(v); - if (match) { - v = match[1] ? match[1] : match[0]; + const setter: AnnotationEventFieldSetter = { key: evts.key, split: evts.split }; + + if (opt.source === AnnotationEventFieldSource.Text) { + setter.text = opt.value; + } else { + const lower = (opt.value || evts.key).toLowerCase(); + setter.field = byName[lower]; + + if (!setter.field && evts.field) { + setter.field = evts.field(frame); + } + } + + if (setter.field || setter.text) { + fields.push(setter); + if (setter.key === 'time') { + hasTime = true; + } else if (setter.key === 'text') { + hasText = true; } } } - if (v !== null && v !== undefined) { - if (f.split && typeof v === 'string') { - v = v.split(','); - } - (anno as any)[f.key] = v; + if (!hasTime || !hasText) { + return []; // throw an error? } - } - events.push(anno); - } + // Add each value to the string + const events: AnnotationEvent[] = []; - return events; + for (let i = 0; i < frame.length; i++) { + const anno: AnnotationEvent = { + type: 'default', + color: 'red', + }; + + for (const f of fields) { + let v: any = undefined; + + if (f.text) { + v = f.text; // TODO support templates! + } else if (f.field) { + v = f.field.values.get(i); + if (v !== undefined && f.regex) { + const match = f.regex.exec(v); + if (match) { + v = match[1] ? match[1] : match[0]; + } + } + } + + if (v !== null && v !== undefined) { + if (f.split && typeof v === 'string') { + v = v.split(','); + } + (anno as any)[f.key] = v; + } + } + + events.push(anno); + } + + return events; + }) + ); } diff --git a/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx b/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx index 4fe27e03650..68f5e6b534a 100644 --- a/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx +++ b/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx @@ -1,4 +1,5 @@ import React, { PureComponent } from 'react'; +import AutoSizer from 'react-virtualized-auto-sizer'; import { applyFieldOverrides, applyRawFieldOverrides, @@ -12,7 +13,6 @@ import { } from '@grafana/data'; import { Button, Container, Field, HorizontalGroup, Icon, Select, Switch, Table, VerticalGroup } from '@grafana/ui'; import { selectors } from '@grafana/e2e-selectors'; -import AutoSizer from 'react-virtualized-auto-sizer'; import { getPanelInspectorStyles } from './styles'; import { config } from 'app/core/config'; @@ -38,6 +38,7 @@ interface State { transformId: DataTransformerID; dataFrameIndex: number; transformationOptions: Array>; + transformedData: DataFrame[]; } export class InspectDataTab extends PureComponent { @@ -49,9 +50,38 @@ export class InspectDataTab extends PureComponent { dataFrameIndex: 0, transformId: DataTransformerID.noop, transformationOptions: buildTransformationOptions(), + transformedData: props.data ?? [], }; } + componentDidUpdate(prevProps: Props, prevState: State) { + if (!this.props.data) { + this.setState({ transformedData: [] }); + return; + } + + if (this.props.options.withTransforms) { + this.setState({ transformedData: this.props.data }); + return; + } + + if (prevProps.data !== this.props.data || prevState.transformId !== this.state.transformId) { + const currentTransform = this.state.transformationOptions.find(item => item.value === this.state.transformId); + + if (currentTransform && currentTransform.transformer.id !== DataTransformerID.noop) { + const selectedDataFrame = this.state.selectedDataFrame; + const dataFrameIndex = this.state.dataFrameIndex; + const subscription = transformDataFrame([currentTransform.transformer], this.props.data).subscribe(data => { + this.setState({ transformedData: data, selectedDataFrame, dataFrameIndex }, () => subscription.unsubscribe()); + }); + return; + } + + this.setState({ transformedData: this.props.data }); + return; + } + } + exportCsv = (dataFrame: DataFrame) => { const { panel } = this.props; const { transformId } = this.state; @@ -73,47 +103,11 @@ export class InspectDataTab extends PureComponent { dataFrameIndex: typeof item.value === 'number' ? item.value : 0, selectedDataFrame: item.value!, }); - - this.props.onOptionsChange({ - ...this.props.options, - }); }; - getTransformedData(): DataFrame[] { - const { transformId, transformationOptions } = this.state; - const { data } = this.props; - - if (!data) { - return []; - } - - const currentTransform = transformationOptions.find(item => item.value === transformId); - - if (currentTransform && currentTransform.transformer.id !== DataTransformerID.noop) { - return transformDataFrame([currentTransform.transformer], data); - } - return data; - } - getProcessedData(): DataFrame[] { const { options } = this.props; - let data = this.props.data; - - if (!data) { - return []; - } - - if (this.state.transformId !== DataTransformerID.noop) { - data = this.getTransformedData(); - } - - // In case the transform removes the currently selected data frame - if (!data[this.state.dataFrameIndex]) { - this.setState({ - dataFrameIndex: 0, - selectedDataFrame: 0, - }); - } + const data = this.state.transformedData; if (!options.withFieldConfig) { return applyRawFieldOverrides(data); @@ -265,9 +259,9 @@ export class InspectDataTab extends PureComponent { return
No Data
; } - if (!dataFrames[dataFrameIndex]) { - return
Could not find the Data Frame
; - } + // let's make sure we don't try to render a frame that doesn't exists + const index = !dataFrames[dataFrameIndex] ? 0 : dataFrameIndex; + const data = dataFrames[index]; return (
@@ -292,7 +286,7 @@ export class InspectDataTab extends PureComponent { return (
- +
); }} diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx index 929b8c21a39..7d7eb1cbd82 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx @@ -1,29 +1,80 @@ -import React, { useContext } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; +import { mergeMap } from 'rxjs/operators'; import { css } from 'emotion'; -import { Icon, JSONFormatter, ThemeContext } from '@grafana/ui'; -import { GrafanaTheme, DataFrame } from '@grafana/data'; +import { Icon, JSONFormatter, useStyles } from '@grafana/ui'; +import { + DataFrame, + DataTransformerConfig, + GrafanaTheme, + transformDataFrame, + TransformerRegistyItem, +} from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; +import { TransformationsEditorTransformation } from './types'; + interface TransformationEditorProps { - name: string; - description?: string; - editor?: JSX.Element; - input: DataFrame[]; - output?: DataFrame[]; debugMode?: boolean; + index: number; + data: DataFrame[]; + uiConfig: TransformerRegistyItem; + configs: TransformationsEditorTransformation[]; + onChange: (index: number, config: DataTransformerConfig) => void; } -export const TransformationEditor = ({ editor, input, output, debugMode, name }: TransformationEditorProps) => { - const theme = useContext(ThemeContext); - const styles = getStyles(theme); +export const TransformationEditor = ({ + debugMode, + index, + data, + uiConfig, + configs, + onChange, +}: TransformationEditorProps) => { + const styles = useStyles(getStyles); + const [input, setInput] = useState([]); + const [output, setOutput] = useState([]); + const config = useMemo(() => configs[index], [configs, index]); + + useEffect(() => { + const inputTransforms = configs.slice(0, index).map(t => t.transformation); + const outputTransforms = configs.slice(index, index + 1).map(t => t.transformation); + const inputSubscription = transformDataFrame(inputTransforms, data).subscribe(setInput); + const outputSubscription = transformDataFrame(inputTransforms, data) + .pipe(mergeMap(before => transformDataFrame(outputTransforms, before))) + .subscribe(setOutput); + + return function unsubscribe() { + inputSubscription.unsubscribe(); + outputSubscription.unsubscribe(); + }; + }, [index, data, configs]); + + const editor = useMemo( + () => + React.createElement(uiConfig.editor, { + options: { ...uiConfig.transformation.defaultOptions, ...config.transformation.options }, + input, + onChange: (opts: any) => { + onChange(index, { id: config.transformation.id, options: opts }); + }, + }), + [ + uiConfig.editor, + uiConfig.transformation.defaultOptions, + config.transformation.id, + config.transformation.options, + input, + onChange, + ] + ); return ( -
+
{editor} {debugMode && (
Transformation input data
diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx index b0049f149d5..310d23e2155 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx @@ -1,27 +1,30 @@ -import { DataFrame } from '@grafana/data'; import React, { useState } from 'react'; +import { DataFrame, DataTransformerConfig, TransformerRegistyItem } from '@grafana/data'; import { HorizontalGroup } from '@grafana/ui'; + import { TransformationEditor } from './TransformationEditor'; import { QueryOperationRow } from 'app/core/components/QueryOperationRow/QueryOperationRow'; import { QueryOperationAction } from 'app/core/components/QueryOperationRow/QueryOperationAction'; +import { TransformationsEditorTransformation } from './types'; interface TransformationOperationRowProps { id: string; index: number; - name: string; - description?: string; - input: DataFrame[]; - output: DataFrame[]; - editor?: JSX.Element; - onRemove: () => void; + data: DataFrame[]; + uiConfig: TransformerRegistyItem; + configs: TransformationsEditorTransformation[]; + onRemove: (index: number) => void; + onChange: (index: number, config: DataTransformerConfig) => void; } export const TransformationOperationRow: React.FC = ({ - children, onRemove, index, id, - ...props + data, + configs, + uiConfig, + onChange, }) => { const [showDebug, setShowDebug] = useState(false); @@ -37,14 +40,21 @@ export const TransformationOperationRow: React.FC - + onRemove(index)} /> ); }; return ( - - + + ); }; diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRows.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRows.tsx new file mode 100644 index 00000000000..6477e0ee920 --- /dev/null +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRows.tsx @@ -0,0 +1,43 @@ +import React from 'react'; +import { DataFrame, DataTransformerConfig, standardTransformersRegistry } from '@grafana/data'; + +import { TransformationOperationRow } from './TransformationOperationRow'; +import { TransformationsEditorTransformation } from './types'; + +interface TransformationOperationRowsProps { + data: DataFrame[]; + configs: TransformationsEditorTransformation[]; + onRemove: (index: number) => void; + onChange: (index: number, config: DataTransformerConfig) => void; +} + +export const TransformationOperationRows: React.FC = ({ + data, + onChange, + onRemove, + configs, +}) => { + return ( + <> + {configs.map((t, i) => { + const uiConfig = standardTransformersRegistry.getIfExists(t.transformation.id); + if (!uiConfig) { + return null; + } + + return ( + + ); + })} + + ); +}; diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationsEditor.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationsEditor.tsx index c3ff05d5f21..a50aa36e8c4 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationsEditor.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationsEditor.tsx @@ -18,9 +18,7 @@ import { PanelData, SelectableValue, standardTransformersRegistry, - transformDataFrame, } from '@grafana/data'; -import { TransformationOperationRow } from './TransformationOperationRow'; import { Card, CardProps } from '../../../../core/components/Card/Card'; import { css } from 'emotion'; import { selectors } from '@grafana/e2e-selectors'; @@ -28,6 +26,8 @@ import { Unsubscribable } from 'rxjs'; import { PanelModel } from '../../state'; import { getDocsLink } from 'app/core/utils/docsLinks'; import { DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd'; +import { TransformationOperationRows } from './TransformationOperationRows'; +import { TransformationsEditorTransformation } from './types'; import { PanelNotSupported } from '../PanelEditor/PanelNotSupported'; import { AppNotificationSeverity } from '../../../../types'; @@ -35,11 +35,6 @@ interface TransformationsEditorProps { panel: PanelModel; } -interface TransformationsEditorTransformation { - transformation: DataTransformerConfig; - id: string; -} - interface State { data: DataFrame[]; transformations: TransformationsEditorTransformation[]; @@ -197,53 +192,12 @@ export class TransformationsEditor extends React.PureComponent { return (
- {transformations.map((t, i) => { - // Transformations are not identified uniquely by any property apart from array index. - // For drag and drop to work we need to generate unique ids. This record stores counters for each transformation type - // based on which ids are generated - let editor; - - const transformationUI = standardTransformersRegistry.getIfExists(t.transformation.id); - if (!transformationUI) { - return null; - } - - const input = transformDataFrame( - transformations.slice(0, i).map(t => t.transformation), - data - ); - const output = transformDataFrame( - transformations.slice(i).map(t => t.transformation), - input - ); - - if (transformationUI) { - editor = React.createElement(transformationUI.editor, { - options: { ...transformationUI.transformation.defaultOptions, ...t.transformation.options }, - input, - onChange: (options: any) => { - this.onTransformationChange(i, { - id: t.transformation.id, - options, - }); - }, - }); - } - - return ( - this.onTransformationRemove(i)} - editor={editor} - name={transformationUI.name} - description={transformationUI.description} - /> - ); - })} + {provided.placeholder}
); diff --git a/public/app/features/dashboard/components/TransformationsEditor/types.ts b/public/app/features/dashboard/components/TransformationsEditor/types.ts new file mode 100644 index 00000000000..50ff63d2fc8 --- /dev/null +++ b/public/app/features/dashboard/components/TransformationsEditor/types.ts @@ -0,0 +1,6 @@ +import { DataTransformerConfig } from '@grafana/data'; + +export interface TransformationsEditorTransformation { + transformation: DataTransformerConfig; + id: string; +} diff --git a/public/app/features/dashboard/state/PanelQueryRunner.ts b/public/app/features/dashboard/state/PanelQueryRunner.ts index 36ad39e82da..4f1cf8818a8 100644 --- a/public/app/features/dashboard/state/PanelQueryRunner.ts +++ b/public/app/features/dashboard/state/PanelQueryRunner.ts @@ -1,31 +1,31 @@ // Libraries import { cloneDeep } from 'lodash'; -import { ReplaySubject, Unsubscribable, Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { MonoTypeOperatorFunction, Observable, of, ReplaySubject, Unsubscribable } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; // Services & Utils import { getTemplateSrv } from '@grafana/runtime'; import { getDatasourceSrv } from 'app/features/plugins/datasource_srv'; -import { runRequest, preProcessPanelData } from './runRequest'; -import { runSharedRequest, isSharedDashboardQuery } from '../../../plugins/datasource/dashboard'; +import { preProcessPanelData, runRequest } from './runRequest'; +import { isSharedDashboardQuery, runSharedRequest } from '../../../plugins/datasource/dashboard'; // Types import { - PanelData, - DataQuery, + applyFieldOverrides, CoreApp, + DataConfigSource, + DataQuery, DataQueryRequest, DataSourceApi, DataSourceJsonData, - TimeRange, DataTransformerConfig, - transformDataFrame, - ScopedVars, - applyFieldOverrides, - DataConfigSource, - TimeZone, LoadingState, + PanelData, rangeUtil, + ScopedVars, + TimeRange, + TimeZone, + transformDataFrame, } from '@grafana/data'; export interface QueryRunnerOptions< @@ -75,21 +75,10 @@ export class PanelQueryRunner { const { withFieldConfig, withTransforms } = options; return this.subject.pipe( + this.getTransformationsStream(withTransforms), map((data: PanelData) => { let processedData = data; - // Apply transformation - if (withTransforms) { - const transformations = this.dataConfigSource.getTransformations(); - - if (transformations && transformations.length > 0) { - processedData = { - ...processedData, - series: transformDataFrame(transformations, data.series), - }; - } - } - if (withFieldConfig) { // Apply field defaults & overrides const fieldConfig = this.dataConfigSource.getFieldOverrideOptions(); @@ -113,6 +102,25 @@ export class PanelQueryRunner { ); } + private getTransformationsStream = (withTransforms: boolean): MonoTypeOperatorFunction => { + return inputStream => + inputStream.pipe( + mergeMap(data => { + if (!withTransforms) { + return of(data); + } + + const transformations = this.dataConfigSource.getTransformations(); + + if (!transformations || transformations.length === 0) { + return of(data); + } + + return transformDataFrame(transformations, data.series).pipe(map(series => ({ ...data, series }))); + }) + ); + }; + async run(options: QueryRunnerOptions) { const { queries, diff --git a/public/app/features/explore/state/actionTypes.ts b/public/app/features/explore/state/actionTypes.ts index dcdc6673f63..f1623b7a5b3 100644 --- a/public/app/features/explore/state/actionTypes.ts +++ b/public/app/features/explore/state/actionTypes.ts @@ -10,12 +10,11 @@ import { HistoryItem, LoadingState, LogLevel, - PanelData, + LogsDedupStrategy, QueryFixAction, TimeRange, - LogsDedupStrategy, } from '@grafana/data'; -import { ExploreId, ExploreItemState } from 'app/types/explore'; +import { ExploreId, ExploreItemState, ExplorePanelData } from 'app/types/explore'; export interface AddQueryRowPayload { exploreId: ExploreId; @@ -82,7 +81,7 @@ export interface ModifyQueriesPayload { export interface QueryEndedPayload { exploreId: ExploreId; - response: PanelData; + response: ExplorePanelData; } export interface QueryStoreSubscriptionPayload { diff --git a/public/app/features/explore/state/actions.ts b/public/app/features/explore/state/actions.ts index aa4f3e2dca6..a93545e7678 100644 --- a/public/app/features/explore/state/actions.ts +++ b/public/app/features/explore/state/actions.ts @@ -9,14 +9,14 @@ import { DataQuery, DataSourceApi, dateTimeForTimeZone, + ExploreUrlState, isDateTime, LoadingState, + LogsDedupStrategy, PanelData, QueryFixAction, RawTimeRange, TimeRange, - ExploreUrlState, - LogsDedupStrategy, } from '@grafana/data'; // Services & Utils import store from 'app/core/store'; @@ -40,17 +40,21 @@ import { import { addToRichHistory, deleteAllFromRichHistory, - updateStarredInRichHistory, - updateCommentInRichHistory, deleteQueryInRichHistory, getRichHistory, + updateCommentInRichHistory, + updateStarredInRichHistory, } from 'app/core/utils/richHistory'; // Types -import { ExploreItemState, ThunkResult } from 'app/types'; +import { ThunkResult } from 'app/types'; -import { ExploreId, QueryOptions } from 'app/types/explore'; +import { ExploreId, ExploreItemState, QueryOptions } from 'app/types/explore'; import { addQueryRowAction, + cancelQueriesAction, + changeDedupStrategyAction, + ChangeDedupStrategyPayload, + changeLoadingStateAction, changeQueryAction, changeRangeAction, changeRefreshIntervalAction, @@ -59,7 +63,6 @@ import { ChangeSizePayload, clearQueriesAction, historyUpdatedAction, - richHistoryUpdatedAction, initializeExploreAction, loadDatasourceMissingAction, loadDatasourcePendingAction, @@ -69,6 +72,7 @@ import { queriesImportedAction, queryStoreSubscriptionAction, queryStreamUpdatedAction, + richHistoryUpdatedAction, scanStartAction, scanStopAction, setQueriesAction, @@ -77,19 +81,21 @@ import { splitOpenAction, syncTimesAction, updateDatasourceInstanceAction, - changeLoadingStateAction, - cancelQueriesAction, - changeDedupStrategyAction, - ChangeDedupStrategyPayload, } from './actionTypes'; import { getTimeZone } from 'app/features/profile/state/selectors'; import { getShiftedTimeRange } from 'app/core/utils/timePicker'; import { updateLocation } from '../../../core/actions'; import { getTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv'; import { preProcessPanelData, runRequest } from '../../dashboard/state/runRequest'; -import { PanelModel, DashboardModel } from 'app/features/dashboard/state'; +import { DashboardModel, PanelModel } from 'app/features/dashboard/state'; import { getExploreDatasources } from './selectors'; import { serializeStateToUrlParam } from '@grafana/data/src/utils/url'; +import { + decorateWithGraphLogsTraceAndTable, + decorateWithGraphResult, + decorateWithLogsResult, + decorateWithTableResult, +} from '../utils/decorators'; /** * Adds a query row after the row with the given index. @@ -466,9 +472,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult => { // rendering. In case this is optimized this can be tweaked, but also it should be only as fast as user // actually can see what is happening. live ? throttleTime(500) : identity, - map((data: PanelData) => preProcessPanelData(data, queryResponse)) + map((data: PanelData) => preProcessPanelData(data, queryResponse)), + decorateWithGraphLogsTraceAndTable(getState().explore[exploreId].datasourceInstance), + decorateWithGraphResult(), + decorateWithTableResult(), + decorateWithLogsResult(getState().explore[exploreId]) ) - .subscribe((data: PanelData) => { + .subscribe(data => { if (!data.error && firstResponse) { // Side-effect: Saving history in localstorage const nextHistory = updateHistory(history, datasourceId, queries); diff --git a/public/app/features/explore/state/reducers.ts b/public/app/features/explore/state/reducers.ts index e1ec97a10d9..d0ab8bcb911 100644 --- a/public/app/features/explore/state/reducers.ts +++ b/public/app/features/explore/state/reducers.ts @@ -3,15 +3,14 @@ import { AnyAction } from 'redux'; import { PayloadAction } from '@reduxjs/toolkit'; import { DataQuery, + DataQueryErrorType, DefaultTimeRange, LoadingState, + LogsDedupStrategy, PanelData, PanelEvents, - TimeZone, - toLegacyResponseData, - LogsDedupStrategy, sortLogsResult, - DataQueryErrorType, + toLegacyResponseData, } from '@grafana/data'; import { RefreshPicker } from '@grafana/ui'; import { LocationUpdate } from '@grafana/runtime'; @@ -27,6 +26,8 @@ import { import { ExploreId, ExploreItemState, ExploreState, ExploreUpdateState } from 'app/types/explore'; import { addQueryRowAction, + cancelQueriesAction, + changeDedupStrategyAction, changeLoadingStateAction, changeQueryAction, changeRangeAction, @@ -35,7 +36,6 @@ import { clearQueriesAction, highlightLogsExpressionAction, historyUpdatedAction, - richHistoryUpdatedAction, initializeExploreAction, loadDatasourceMissingAction, loadDatasourcePendingAction, @@ -48,6 +48,7 @@ import { removeQueryRowAction, resetExploreAction, ResetExplorePayload, + richHistoryUpdatedAction, scanStartAction, scanStopAction, setPausedStateAction, @@ -59,10 +60,7 @@ import { syncTimesAction, toggleLogLevelAction, updateDatasourceInstanceAction, - cancelQueriesAction, - changeDedupStrategyAction, } from './actionTypes'; -import { ResultProcessor } from '../utils/ResultProcessor'; import { updateLocation } from '../../../core/actions'; import { Emitter } from 'app/core/core'; @@ -464,7 +462,7 @@ export const processQueryResponse = ( action: PayloadAction ): ExploreItemState => { const { response } = action.payload; - const { request, state: loadingState, series, error } = response; + const { request, state: loadingState, series, error, graphResult, logsResult, tableResult, traceFrames } = response; if (error) { if (error.type === DataQueryErrorType.Timeout) { @@ -496,10 +494,6 @@ export const processQueryResponse = ( } const latency = request.endTime ? request.endTime - request.startTime : 0; - const processor = new ResultProcessor(state, series, request.intervalMs, request.timezone as TimeZone); - const graphResult = processor.getGraphResult(); - const tableResult = processor.getTableResult(); - const logsResult = processor.getLogsResult(); // Send legacy data to Angular editors if (state.datasourceInstance?.components?.QueryCtrl) { @@ -520,7 +514,7 @@ export const processQueryResponse = ( showLogs: !!logsResult, showMetrics: !!graphResult, showTable: !!tableResult, - showTrace: !!processor.traceFrames.length, + showTrace: !!traceFrames.length, }; }; diff --git a/public/app/features/explore/utils/ResultProcessor.test.ts b/public/app/features/explore/utils/ResultProcessor.test.ts deleted file mode 100644 index 3778049d436..00000000000 --- a/public/app/features/explore/utils/ResultProcessor.test.ts +++ /dev/null @@ -1,323 +0,0 @@ -jest.mock('@grafana/data/src/datetime/formatter', () => ({ - dateTimeFormat: () => 'format() jest mocked', - dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked', -})); - -import { ResultProcessor } from './ResultProcessor'; -import { ExploreItemState } from 'app/types/explore'; -import TableModel from 'app/core/table_model'; -import { FieldType, LogRowModel, TimeSeries, toDataFrame, ArrayVector } from '@grafana/data'; - -const testContext = (options: any = {}) => { - const timeSeries = toDataFrame({ - name: 'A-series', - refId: 'A', - meta: { - preferredVisualisationType: 'graph', - }, - fields: [ - { name: 'time', type: FieldType.time, values: [100, 200, 300] }, - { name: 'A-series', type: FieldType.number, values: [4, 5, 6] }, - ], - }); - - const table = toDataFrame({ - name: 'table-res', - refId: 'A', - fields: [ - { name: 'value', type: FieldType.number, values: [4, 5, 6] }, - { name: 'time', type: FieldType.time, values: [100, 100, 100] }, - { name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] }, - { name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] }, - ], - }); - - const emptyTable = toDataFrame({ name: 'empty-table', refId: 'A', fields: [] }); - - const logs = toDataFrame({ - name: 'logs-res', - refId: 'A', - fields: [ - { name: 'value', type: FieldType.number, values: [4, 5, 6] }, - { name: 'time', type: FieldType.time, values: [100, 100, 100] }, - { name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] }, - { name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] }, - ], - meta: { preferredVisualisationType: 'logs' }, - }); - - const defaultOptions = { - dataFrames: [timeSeries, table, emptyTable, logs], - graphResult: [] as TimeSeries[], - tableResult: new TableModel(), - logsResult: { hasUniqueLabels: false, rows: [] as LogRowModel[] }, - }; - - const combinedOptions = { ...defaultOptions, ...options }; - - const state = ({ - graphResult: combinedOptions.graphResult, - tableResult: combinedOptions.tableResult, - logsResult: combinedOptions.logsResult, - queryIntervals: { intervalMs: 10 }, - } as any) as ExploreItemState; - - const resultProcessor = new ResultProcessor(state, combinedOptions.dataFrames, 60000, 'utc'); - - return { - dataFrames: combinedOptions.dataFrames, - resultProcessor, - }; -}; - -describe('ResultProcessor', () => { - describe('constructed without result', () => { - describe('when calling getGraphResult', () => { - it('then it should return null', () => { - const { resultProcessor } = testContext({ dataFrames: [] }); - const theResult = resultProcessor.getGraphResult(); - - expect(theResult).toEqual(null); - }); - }); - - describe('when calling getTableResult', () => { - it('then it should return null', () => { - const { resultProcessor } = testContext({ dataFrames: [] }); - const theResult = resultProcessor.getTableResult(); - - expect(theResult).toEqual(null); - }); - }); - - describe('when calling getLogsResult', () => { - it('then it should return null', () => { - const { resultProcessor } = testContext({ dataFrames: [] }); - const theResult = resultProcessor.getLogsResult(); - - expect(theResult).toBeNull(); - }); - }); - }); - - describe('constructed with a result that is a DataQueryResponse', () => { - describe('when calling getGraphResult', () => { - it('then it should return correct graph result', () => { - const { resultProcessor, dataFrames } = testContext(); - const timeField = dataFrames[0].fields[0]; - const valueField = dataFrames[0].fields[1]; - const theResult = resultProcessor.getGraphResult(); - - expect(theResult![0]).toEqual({ - label: 'A-series', - color: '#7EB26D', - data: [ - [100, 4], - [200, 5], - [300, 6], - ], - info: [], - isVisible: true, - yAxis: { - index: 1, - }, - seriesIndex: 0, - timeField, - valueField, - timeStep: 100, - }); - }); - }); - - describe('when calling getTableResult', () => { - it('then it should return correct table result', () => { - const { resultProcessor } = testContext(); - let theResult = resultProcessor.getTableResult(); - - expect(theResult?.fields[0].name).toEqual('value'); - expect(theResult?.fields[1].name).toEqual('time'); - expect(theResult?.fields[2].name).toEqual('tsNs'); - expect(theResult?.fields[3].name).toEqual('message'); - expect(theResult?.fields[1].display).not.toBeNull(); - expect(theResult?.length).toBe(3); - - // Same data though a DataFrame - theResult = toDataFrame( - new TableModel({ - columns: [ - { text: 'value', type: 'number' }, - { text: 'time', type: 'time' }, - { text: 'tsNs', type: 'time' }, - { text: 'message', type: 'string' }, - ], - rows: [ - [4, 100, '100000000', 'this is a message'], - [5, 200, '100000000', 'second message'], - [6, 300, '100000000', 'third'], - ], - type: 'table', - }) - ); - expect(theResult.fields[0].name).toEqual('value'); - expect(theResult.fields[1].name).toEqual('time'); - expect(theResult.fields[2].name).toEqual('tsNs'); - expect(theResult.fields[3].name).toEqual('message'); - expect(theResult.fields[1].display).not.toBeNull(); - expect(theResult.length).toBe(3); - }); - - it('should do join transform if all series are timeseries', () => { - const { resultProcessor } = testContext({ - dataFrames: [ - toDataFrame({ - name: 'A-series', - refId: 'A', - fields: [ - { name: 'Time', type: FieldType.time, values: [100, 200, 300] }, - { name: 'A-series', type: FieldType.number, values: [4, 5, 6] }, - ], - }), - toDataFrame({ - name: 'B-series', - refId: 'B', - fields: [ - { name: 'Time', type: FieldType.time, values: [100, 200, 300] }, - { name: 'B-series', type: FieldType.number, values: [4, 5, 6] }, - ], - }), - ], - }); - - let result = resultProcessor.getTableResult()!; - - expect(result.fields[0].name).toBe('Time'); - expect(result.fields[1].name).toBe('A-series'); - expect(result.fields[2].name).toBe('B-series'); - expect(result.fields[0].values.toArray()).toEqual([100, 200, 300]); - expect(result.fields[1].values.toArray()).toEqual([4, 5, 6]); - expect(result.fields[2].values.toArray()).toEqual([4, 5, 6]); - }); - - it('should not override fields display property when filled', () => { - const { resultProcessor, dataFrames } = testContext({ - dataFrames: [ - toDataFrame({ - name: 'A-series', - refId: 'A', - fields: [{ name: 'Text', type: FieldType.string, values: ['someText'] }], - }), - ], - }); - const displayFunctionMock = jest.fn(); - dataFrames[0].fields[0].display = displayFunctionMock; - - const data = resultProcessor.getTableResult(); - - expect(data?.fields[0].display).toBe(displayFunctionMock); - }); - }); - - describe('when calling getLogsResult', () => { - it('then it should return correct logs result', () => { - const { resultProcessor, dataFrames } = testContext({}); - const logsDataFrame = dataFrames[3]; - - const theResult = resultProcessor.getLogsResult(); - - expect(theResult).toEqual({ - hasUniqueLabels: false, - meta: [], - rows: [ - { - rowIndex: 0, - dataFrame: logsDataFrame, - entry: 'this is a message', - entryFieldIndex: 3, - hasAnsi: false, - labels: {}, - logLevel: 'unknown', - raw: 'this is a message', - searchWords: [] as string[], - timeEpochMs: 100, - timeEpochNs: '100000002', - timeFromNow: 'fromNow() jest mocked', - timeLocal: 'format() jest mocked', - timeUtc: 'format() jest mocked', - uid: '0', - uniqueLabels: {}, - }, - { - rowIndex: 2, - dataFrame: logsDataFrame, - entry: 'third', - entryFieldIndex: 3, - hasAnsi: false, - labels: {}, - logLevel: 'unknown', - raw: 'third', - searchWords: [] as string[], - timeEpochMs: 100, - timeEpochNs: '100000001', - timeFromNow: 'fromNow() jest mocked', - timeLocal: 'format() jest mocked', - timeUtc: 'format() jest mocked', - uid: '2', - uniqueLabels: {}, - }, - { - rowIndex: 1, - dataFrame: logsDataFrame, - entry: 'second message', - entryFieldIndex: 3, - hasAnsi: false, - labels: {}, - logLevel: 'unknown', - raw: 'second message', - searchWords: [] as string[], - timeEpochMs: 100, - timeEpochNs: '100000000', - timeFromNow: 'fromNow() jest mocked', - timeLocal: 'format() jest mocked', - timeUtc: 'format() jest mocked', - uid: '1', - uniqueLabels: {}, - }, - ], - series: [ - { - label: 'unknown', - color: '#8e8e8e', - data: [[0, 3]], - isVisible: true, - yAxis: { - index: 1, - min: 0, - tickDecimals: 0, - }, - seriesIndex: 0, - timeField: { - name: 'Time', - type: 'time', - config: {}, - values: new ArrayVector([0]), - index: 0, - display: expect.anything(), - }, - valueField: { - name: 'unknown', - type: 'number', - config: { unit: undefined, color: '#8e8e8e' }, - values: new ArrayVector([3]), - labels: undefined, - index: 1, - display: expect.anything(), - }, - timeStep: 0, - }, - ], - visibleRange: undefined, - }); - }); - }); - }); -}); diff --git a/public/app/features/explore/utils/ResultProcessor.ts b/public/app/features/explore/utils/ResultProcessor.ts deleted file mode 100644 index c441c79d5f0..00000000000 --- a/public/app/features/explore/utils/ResultProcessor.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { - LogsModel, - GraphSeriesXY, - DataFrame, - FieldType, - TimeZone, - getDisplayProcessor, - PreferredVisualisationType, - standardTransformers, - sortLogsResult, -} from '@grafana/data'; -import { ExploreItemState } from 'app/types/explore'; -import { refreshIntervalToSortOrder } from 'app/core/utils/explore'; -import { dataFrameToLogsModel } from 'app/core/logs_model'; -import { getGraphSeriesModel } from 'app/plugins/panel/graph2/getGraphSeriesModel'; -import { config } from 'app/core/config'; - -export class ResultProcessor { - graphFrames: DataFrame[] = []; - tableFrames: DataFrame[] = []; - logsFrames: DataFrame[] = []; - traceFrames: DataFrame[] = []; - - constructor( - private state: ExploreItemState, - private dataFrames: DataFrame[], - private intervalMs: number, - private timeZone: TimeZone - ) { - this.classifyFrames(); - } - - private classifyFrames() { - for (const frame of this.dataFrames) { - if (shouldShowInVisualisationTypeStrict(frame, 'logs')) { - this.logsFrames.push(frame); - } else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) { - this.graphFrames.push(frame); - } else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) { - this.traceFrames.push(frame); - } else if (shouldShowInVisualisationTypeStrict(frame, 'table')) { - this.tableFrames.push(frame); - } else if (isTimeSeries(frame, this.state.datasourceInstance?.meta.id)) { - if (shouldShowInVisualisationType(frame, 'graph')) { - this.graphFrames.push(frame); - } - if (shouldShowInVisualisationType(frame, 'table')) { - this.tableFrames.push(frame); - } - } else { - // We fallback to table if we do not have any better meta info about the dataframe. - this.tableFrames.push(frame); - } - } - } - - getGraphResult(): GraphSeriesXY[] | null { - if (this.graphFrames.length === 0) { - return null; - } - - return getGraphSeriesModel( - this.graphFrames, - this.timeZone, - {}, - { showBars: false, showLines: true, showPoints: false }, - { asTable: false, isVisible: true, placement: 'under' } - ); - } - - getTableResult(): DataFrame | null { - if (this.tableFrames.length === 0) { - return null; - } - - this.tableFrames.sort((frameA: DataFrame, frameB: DataFrame) => { - const frameARefId = frameA.refId!; - const frameBRefId = frameB.refId!; - - if (frameARefId > frameBRefId) { - return 1; - } - if (frameARefId < frameBRefId) { - return -1; - } - return 0; - }); - - const hasOnlyTimeseries = this.tableFrames.every(df => isTimeSeries(df)); - - // If we have only timeseries we do join on default time column which makes more sense. If we are showing - // non timeseries or some mix of data we are not trying to join on anything and just try to merge them in - // single table, which may not make sense in most cases, but it's up to the user to query something sensible. - const transformer = hasOnlyTimeseries - ? standardTransformers.seriesToColumnsTransformer.transformer({}) - : standardTransformers.mergeTransformer.transformer({}); - - const data = transformer(this.tableFrames)[0]; - - // set display processor - for (const field of data.fields) { - field.display = - field.display ?? - getDisplayProcessor({ - field, - theme: config.theme, - timeZone: this.timeZone, - }); - } - - return data; - } - - getLogsResult(): LogsModel | null { - if (this.logsFrames.length === 0) { - return null; - } - - const newResults = dataFrameToLogsModel(this.logsFrames, this.intervalMs, this.timeZone, this.state.absoluteRange); - const sortOrder = refreshIntervalToSortOrder(this.state.refreshInterval); - const sortedNewResults = sortLogsResult(newResults, sortOrder); - const rows = sortedNewResults.rows; - const series = sortedNewResults.series; - return { ...sortedNewResults, rows, series }; - } -} - -function isTimeSeries(frame: DataFrame, datasource?: string): boolean { - // TEMP: Temporary hack. Remove when logs/metrics unification is done - if (datasource && datasource === 'cloudwatch') { - return isTimeSeriesCloudWatch(frame); - } - - if (frame.fields.length === 2) { - if (frame.fields[0].type === FieldType.time) { - return true; - } - } - - return false; -} - -function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) { - if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) { - return false; - } - - return true; -} - -function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) { - return frame.meta?.preferredVisualisationType === visualisation; -} - -// TEMP: Temporary hack. Remove when logs/metrics unification is done -function isTimeSeriesCloudWatch(frame: DataFrame): boolean { - return ( - frame.fields.some(field => field.type === FieldType.time) && - frame.fields.some(field => field.type === FieldType.number) - ); -} diff --git a/public/app/features/explore/utils/decorators.test.ts b/public/app/features/explore/utils/decorators.test.ts new file mode 100644 index 00000000000..d913531c953 --- /dev/null +++ b/public/app/features/explore/utils/decorators.test.ts @@ -0,0 +1,534 @@ +jest.mock('@grafana/data/src/datetime/formatter', () => ({ + dateTimeFormat: () => 'format() jest mocked', + dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked', +})); + +import { of } from 'rxjs'; +import { + ArrayVector, + DataFrame, + DataQueryRequest, + DataSourceApi, + FieldType, + LoadingState, + observableTester, + PanelData, + TimeRange, + toDataFrame, +} from '@grafana/data'; + +import { + decorateWithGraphLogsTraceAndTable, + decorateWithGraphResult, + decorateWithLogsResult, + decorateWithTableResult, +} from './decorators'; +import { describe } from '../../../../test/lib/common'; +import { ExploreItemState, ExplorePanelData } from 'app/types'; +import TableModel from 'app/core/table_model'; + +const getTestContext = () => { + const timeSeries = toDataFrame({ + name: 'A-series', + refId: 'A', + meta: { + preferredVisualisationType: 'graph', + }, + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'A-series', type: FieldType.number, values: [4, 5, 6] }, + ], + }); + + const table = toDataFrame({ + name: 'table-res', + refId: 'A', + fields: [ + { name: 'value', type: FieldType.number, values: [4, 5, 6] }, + { name: 'time', type: FieldType.time, values: [100, 100, 100] }, + { name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] }, + { name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] }, + ], + }); + + const emptyTable = toDataFrame({ name: 'empty-table', refId: 'A', fields: [] }); + + const logs = toDataFrame({ + name: 'logs-res', + refId: 'A', + fields: [ + { name: 'value', type: FieldType.number, values: [4, 5, 6] }, + { name: 'time', type: FieldType.time, values: [100, 100, 100] }, + { name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] }, + { name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] }, + ], + meta: { preferredVisualisationType: 'logs' }, + }); + + return { emptyTable, timeSeries, logs, table }; +}; + +const createExplorePanelData = (args: Partial): ExplorePanelData => { + const defaults: ExplorePanelData = { + series: [], + timeRange: ({} as unknown) as TimeRange, + state: LoadingState.Done, + graphFrames: [], + graphResult: (undefined as unknown) as null, + logsFrames: [], + logsResult: (undefined as unknown) as null, + tableFrames: [], + tableResult: (undefined as unknown) as null, + traceFrames: [], + }; + + return { ...defaults, ...args }; +}; + +describe('decorateWithGraphLogsTraceAndTable', () => { + describe('when used without error', () => { + it('then the result should be correct', done => { + const { table, logs, timeSeries, emptyTable } = getTestContext(); + const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi; + const series = [table, logs, timeSeries, emptyTable]; + const panelData: PanelData = { + series, + state: LoadingState.Done, + timeRange: ({} as unknown) as TimeRange, + }; + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)), + expect: value => { + expect(value).toEqual({ + series, + state: LoadingState.Done, + timeRange: {}, + graphFrames: [timeSeries], + tableFrames: [table, emptyTable], + logsFrames: [logs], + traceFrames: [], + graphResult: null, + tableResult: null, + logsResult: null, + }); + }, + done, + }); + }); + }); + + describe('when used without frames', () => { + it('then the result should be correct', done => { + const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi; + const series: DataFrame[] = []; + const panelData: PanelData = { + series, + state: LoadingState.Done, + timeRange: ({} as unknown) as TimeRange, + }; + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)), + expect: value => { + expect(value).toEqual({ + series: [], + state: LoadingState.Done, + timeRange: {}, + graphFrames: [], + tableFrames: [], + logsFrames: [], + traceFrames: [], + graphResult: null, + tableResult: null, + logsResult: null, + }); + }, + done, + }); + }); + }); + + describe('when used with an error', () => { + it('then the result should be correct', done => { + const { timeSeries, logs, table } = getTestContext(); + const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi; + const series: DataFrame[] = [timeSeries, logs, table]; + const panelData: PanelData = { + series, + error: {}, + state: LoadingState.Error, + timeRange: ({} as unknown) as TimeRange, + }; + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)), + expect: value => { + expect(value).toEqual({ + series: [timeSeries, logs, table], + error: {}, + state: LoadingState.Error, + timeRange: {}, + graphFrames: [], + tableFrames: [], + logsFrames: [], + traceFrames: [], + graphResult: null, + tableResult: null, + logsResult: null, + }); + }, + done, + }); + }); + }); +}); + +describe('decorateWithGraphResult', () => { + describe('when used without error', () => { + it('then the graphResult should be correct', done => { + const { timeSeries } = getTestContext(); + const timeField = timeSeries.fields[0]; + const valueField = timeSeries.fields[1]; + const panelData = createExplorePanelData({ graphFrames: [timeSeries] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphResult()), + expect: panelData => { + expect(panelData.graphResult![0]).toEqual({ + label: 'A-series', + color: '#7EB26D', + data: [ + [100, 4], + [200, 5], + [300, 6], + ], + info: [], + isVisible: true, + yAxis: { + index: 1, + }, + seriesIndex: 0, + timeField, + valueField, + timeStep: 100, + }); + }, + done, + }); + }); + }); + + describe('when used without error but graph frames are empty', () => { + it('then the graphResult should be null', done => { + const panelData = createExplorePanelData({ graphFrames: [] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphResult()), + expect: panelData => { + expect(panelData.graphResult).toBeNull(); + }, + done, + }); + }); + }); + + describe('when used with error', () => { + it('then the graphResult should be null', done => { + const { timeSeries } = getTestContext(); + const panelData = createExplorePanelData({ error: {}, graphFrames: [timeSeries] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithGraphResult()), + expect: panelData => { + expect(panelData.graphResult).toBeNull(); + }, + done, + }); + }); + }); +}); + +describe('decorateWithTableResult', () => { + describe('when used without error', () => { + it('then the tableResult should be correct', done => { + const { table, emptyTable } = getTestContext(); + const panelData = createExplorePanelData({ tableFrames: [table, emptyTable] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithTableResult()), + expect: panelData => { + let theResult = panelData.tableResult; + + expect(theResult?.fields[0].name).toEqual('value'); + expect(theResult?.fields[1].name).toEqual('time'); + expect(theResult?.fields[2].name).toEqual('tsNs'); + expect(theResult?.fields[3].name).toEqual('message'); + expect(theResult?.fields[1].display).not.toBeNull(); + expect(theResult?.length).toBe(3); + + // I don't understand the purpose of the code below, feels like this belongs in toDataFrame tests? + // Same data though a DataFrame + theResult = toDataFrame( + new TableModel({ + columns: [ + { text: 'value', type: 'number' }, + { text: 'time', type: 'time' }, + { text: 'tsNs', type: 'time' }, + { text: 'message', type: 'string' }, + ], + rows: [ + [4, 100, '100000000', 'this is a message'], + [5, 200, '100000000', 'second message'], + [6, 300, '100000000', 'third'], + ], + type: 'table', + }) + ); + expect(theResult.fields[0].name).toEqual('value'); + expect(theResult.fields[1].name).toEqual('time'); + expect(theResult.fields[2].name).toEqual('tsNs'); + expect(theResult.fields[3].name).toEqual('message'); + expect(theResult.fields[1].display).not.toBeNull(); + expect(theResult.length).toBe(3); + }, + done, + }); + }); + + it('should do join transform if all series are timeseries', done => { + const tableFrames = [ + toDataFrame({ + name: 'A-series', + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'A-series', type: FieldType.number, values: [4, 5, 6] }, + ], + }), + toDataFrame({ + name: 'B-series', + refId: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'B-series', type: FieldType.number, values: [4, 5, 6] }, + ], + }), + ]; + const panelData = createExplorePanelData({ tableFrames }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithTableResult()), + expect: panelData => { + const result = panelData.tableResult; + + expect(result?.fields[0].name).toBe('Time'); + expect(result?.fields[1].name).toBe('A-series'); + expect(result?.fields[2].name).toBe('B-series'); + expect(result?.fields[0].values.toArray()).toEqual([100, 200, 300]); + expect(result?.fields[1].values.toArray()).toEqual([4, 5, 6]); + expect(result?.fields[2].values.toArray()).toEqual([4, 5, 6]); + }, + done, + }); + }); + + it('should not override fields display property when filled', done => { + const tableFrames = [ + toDataFrame({ + name: 'A-series', + refId: 'A', + fields: [{ name: 'Text', type: FieldType.string, values: ['someText'] }], + }), + ]; + const displayFunctionMock = jest.fn(); + tableFrames[0].fields[0].display = displayFunctionMock; + + const panelData = createExplorePanelData({ tableFrames }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithTableResult()), + expect: panelData => { + const data = panelData.tableResult; + expect(data?.fields[0].display).toBe(displayFunctionMock); + }, + done, + }); + }); + }); + + describe('when used without error but table frames are empty', () => { + it('then the tableResult should be null', done => { + const panelData = createExplorePanelData({ tableFrames: [] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithTableResult()), + expect: panelData => { + expect(panelData.tableResult).toBeNull(); + }, + done, + }); + }); + }); + + describe('when used with error', () => { + it('then the tableResult should be null', done => { + const { table, emptyTable } = getTestContext(); + const panelData = createExplorePanelData({ error: {}, tableFrames: [table, emptyTable] }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithTableResult()), + expect: panelData => { + expect(panelData.tableResult).toBeNull(); + }, + done, + }); + }); + }); +}); + +describe('decorateWithLogsResult', () => { + describe('when used without error', () => { + it('then the logsResult should be correct', done => { + const { logs } = getTestContext(); + const state = ({ + queryIntervals: { intervalMs: 10 }, + } as unknown) as ExploreItemState; + const request = ({ timezone: 'utc', intervalMs: 60000 } as unknown) as DataQueryRequest; + const panelData = createExplorePanelData({ logsFrames: [logs], request }); + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithLogsResult(state)), + expect: panelData => { + const theResult = panelData.logsResult; + + expect(theResult).toEqual({ + hasUniqueLabels: false, + meta: [], + rows: [ + { + rowIndex: 0, + dataFrame: logs, + entry: 'this is a message', + entryFieldIndex: 3, + hasAnsi: false, + labels: {}, + logLevel: 'unknown', + raw: 'this is a message', + searchWords: [] as string[], + timeEpochMs: 100, + timeEpochNs: '100000002', + timeFromNow: 'fromNow() jest mocked', + timeLocal: 'format() jest mocked', + timeUtc: 'format() jest mocked', + uid: '0', + uniqueLabels: {}, + }, + { + rowIndex: 2, + dataFrame: logs, + entry: 'third', + entryFieldIndex: 3, + hasAnsi: false, + labels: {}, + logLevel: 'unknown', + raw: 'third', + searchWords: [] as string[], + timeEpochMs: 100, + timeEpochNs: '100000001', + timeFromNow: 'fromNow() jest mocked', + timeLocal: 'format() jest mocked', + timeUtc: 'format() jest mocked', + uid: '2', + uniqueLabels: {}, + }, + { + rowIndex: 1, + dataFrame: logs, + entry: 'second message', + entryFieldIndex: 3, + hasAnsi: false, + labels: {}, + logLevel: 'unknown', + raw: 'second message', + searchWords: [] as string[], + timeEpochMs: 100, + timeEpochNs: '100000000', + timeFromNow: 'fromNow() jest mocked', + timeLocal: 'format() jest mocked', + timeUtc: 'format() jest mocked', + uid: '1', + uniqueLabels: {}, + }, + ], + series: [ + { + label: 'unknown', + color: '#8e8e8e', + data: [[0, 3]], + isVisible: true, + yAxis: { + index: 1, + min: 0, + tickDecimals: 0, + }, + seriesIndex: 0, + timeField: { + name: 'Time', + type: 'time', + config: {}, + values: new ArrayVector([0]), + index: 0, + display: expect.anything(), + }, + valueField: { + name: 'unknown', + type: 'number', + config: { unit: undefined, color: '#8e8e8e' }, + values: new ArrayVector([3]), + labels: undefined, + index: 1, + display: expect.anything(), + }, + timeStep: 0, + }, + ], + visibleRange: undefined, + }); + }, + done, + }); + }); + }); + + describe('when used without error but logs frames are empty', () => { + it('then the graphResult should be null', done => { + const panelData = createExplorePanelData({ logsFrames: [] }); + const state = ({} as unknown) as ExploreItemState; + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithLogsResult(state)), + expect: panelData => { + expect(panelData.logsResult).toBeNull(); + }, + done, + }); + }); + }); + + describe('when used with error', () => { + it('then the graphResult should be null', done => { + const { logs } = getTestContext(); + const panelData = createExplorePanelData({ error: {}, logsFrames: [logs] }); + const state = ({} as unknown) as ExploreItemState; + + observableTester().subscribeAndExpectOnNext({ + observable: of(panelData).pipe(decorateWithLogsResult(state)), + expect: panelData => { + expect(panelData.logsResult).toBeNull(); + }, + done, + }); + }); + }); +}); diff --git a/public/app/features/explore/utils/decorators.ts b/public/app/features/explore/utils/decorators.ts new file mode 100644 index 00000000000..c86e7fc7569 --- /dev/null +++ b/public/app/features/explore/utils/decorators.ts @@ -0,0 +1,214 @@ +import { MonoTypeOperatorFunction, of, OperatorFunction } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; +import { + DataFrame, + DataSourceApi, + FieldType, + getDisplayProcessor, + PanelData, + PreferredVisualisationType, + sortLogsResult, + standardTransformers, +} from '@grafana/data'; +import { config } from '@grafana/runtime'; + +import { ExploreItemState, ExplorePanelData } from '../../../types'; +import { getGraphSeriesModel } from '../../../plugins/panel/graph2/getGraphSeriesModel'; +import { dataFrameToLogsModel } from '../../../core/logs_model'; +import { refreshIntervalToSortOrder } from '../../../core/utils/explore'; + +export const decorateWithGraphLogsTraceAndTable = ( + datasourceInstance?: DataSourceApi | null +): OperatorFunction => inputStream => + inputStream.pipe( + map(data => { + if (data.error) { + return { + ...data, + graphFrames: [], + tableFrames: [], + logsFrames: [], + traceFrames: [], + graphResult: null, + tableResult: null, + logsResult: null, + }; + } + + const graphFrames: DataFrame[] = []; + const tableFrames: DataFrame[] = []; + const logsFrames: DataFrame[] = []; + const traceFrames: DataFrame[] = []; + + for (const frame of data.series) { + if (shouldShowInVisualisationTypeStrict(frame, 'logs')) { + logsFrames.push(frame); + } else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) { + graphFrames.push(frame); + } else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) { + traceFrames.push(frame); + } else if (shouldShowInVisualisationTypeStrict(frame, 'table')) { + tableFrames.push(frame); + } else if (isTimeSeries(frame, datasourceInstance?.meta.id)) { + if (shouldShowInVisualisationType(frame, 'graph')) { + graphFrames.push(frame); + } + if (shouldShowInVisualisationType(frame, 'table')) { + tableFrames.push(frame); + } + } else { + // We fallback to table if we do not have any better meta info about the dataframe. + tableFrames.push(frame); + } + } + + return { + ...data, + graphFrames, + tableFrames, + logsFrames, + traceFrames, + graphResult: null, + tableResult: null, + logsResult: null, + }; + }) + ); + +export const decorateWithGraphResult = (): MonoTypeOperatorFunction => inputStream => + inputStream.pipe( + map(data => { + if (data.error) { + return { ...data, graphResult: null }; + } + + const graphResult = + data.graphFrames.length === 0 + ? null + : getGraphSeriesModel( + data.graphFrames, + data.request?.timezone ?? 'browser', + {}, + { showBars: false, showLines: true, showPoints: false }, + { asTable: false, isVisible: true, placement: 'under' } + ); + + return { ...data, graphResult }; + }) + ); + +export const decorateWithTableResult = (): MonoTypeOperatorFunction => inputStream => + inputStream.pipe( + mergeMap(data => { + if (data.error) { + return of({ ...data, tableResult: null }); + } + + if (data.tableFrames.length === 0) { + return of({ ...data, tableResult: null }); + } + + data.tableFrames.sort((frameA: DataFrame, frameB: DataFrame) => { + const frameARefId = frameA.refId!; + const frameBRefId = frameB.refId!; + + if (frameARefId > frameBRefId) { + return 1; + } + if (frameARefId < frameBRefId) { + return -1; + } + return 0; + }); + + const hasOnlyTimeseries = data.tableFrames.every(df => isTimeSeries(df)); + + // If we have only timeseries we do join on default time column which makes more sense. If we are showing + // non timeseries or some mix of data we are not trying to join on anything and just try to merge them in + // single table, which may not make sense in most cases, but it's up to the user to query something sensible. + const transformer = hasOnlyTimeseries + ? of(data.tableFrames).pipe(standardTransformers.seriesToColumnsTransformer.operator({})) + : of(data.tableFrames).pipe(standardTransformers.mergeTransformer.operator({})); + + return transformer.pipe( + map(frames => { + const frame = frames[0]; + + // set display processor + for (const field of frame.fields) { + field.display = + field.display ?? + getDisplayProcessor({ + field, + theme: config.theme, + timeZone: data.request?.timezone ?? 'browser', + }); + } + + return { ...data, tableResult: frame }; + }) + ); + }) + ); + +export const decorateWithLogsResult = ( + state: ExploreItemState +): MonoTypeOperatorFunction => inputStream => + inputStream.pipe( + map(data => { + if (data.error) { + return { ...data, logsResult: null }; + } + + const { absoluteRange, refreshInterval } = state; + if (data.logsFrames.length === 0) { + return { ...data, logsResult: null }; + } + + const timeZone = data.request?.timezone ?? 'browser'; + const intervalMs = data.request?.intervalMs; + const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, absoluteRange); + const sortOrder = refreshIntervalToSortOrder(refreshInterval); + const sortedNewResults = sortLogsResult(newResults, sortOrder); + const rows = sortedNewResults.rows; + const series = sortedNewResults.series; + const logsResult = { ...sortedNewResults, rows, series }; + + return { ...data, logsResult }; + }) + ); + +function isTimeSeries(frame: DataFrame, datasource?: string): boolean { + // TEMP: Temporary hack. Remove when logs/metrics unification is done + if (datasource && datasource === 'cloudwatch') { + return isTimeSeriesCloudWatch(frame); + } + + if (frame.fields.length === 2) { + if (frame.fields[0].type === FieldType.time) { + return true; + } + } + + return false; +} + +function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) { + if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) { + return false; + } + + return true; +} + +function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) { + return frame.meta?.preferredVisualisationType === visualisation; +} + +// TEMP: Temporary hack. Remove when logs/metrics unification is done +function isTimeSeriesCloudWatch(frame: DataFrame): boolean { + return ( + frame.fields.some(field => field.type === FieldType.time) && + frame.fields.some(field => field.type === FieldType.number) + ); +} diff --git a/public/app/plugins/datasource/loki/datasource.test.ts b/public/app/plugins/datasource/loki/datasource.test.ts index 2071a657633..978dda175b6 100644 --- a/public/app/plugins/datasource/loki/datasource.test.ts +++ b/public/app/plugins/datasource/loki/datasource.test.ts @@ -8,6 +8,7 @@ import { DataQueryResponse, dateTime, FieldCache, + observableTester, TimeRange, } from '@grafana/data'; import { BackendSrvRequest, FetchResponse } from '@grafana/runtime'; @@ -19,7 +20,6 @@ import { TemplateSrv } from 'app/features/templating/template_srv'; import { backendSrv } from 'app/core/services/backend_srv'; import { CustomVariableModel } from '../../../features/variables/types'; import { initialCustomVariableModelState } from '../../../features/variables/custom/reducer'; -import { observableTester } from '../../../../test/helpers/observableTester'; import { expect } from '../../../../test/lib/common'; import { makeMockLokiDatasource } from './mocks'; diff --git a/public/app/plugins/datasource/postgres/specs/datasource.test.ts b/public/app/plugins/datasource/postgres/specs/datasource.test.ts index 22d885f7d9f..c26e8fd2239 100644 --- a/public/app/plugins/datasource/postgres/specs/datasource.test.ts +++ b/public/app/plugins/datasource/postgres/specs/datasource.test.ts @@ -1,14 +1,13 @@ import { of } from 'rxjs'; import { TestScheduler } from 'rxjs/testing'; import { FetchResponse } from '@grafana/runtime'; -import { dateTime, toUtc } from '@grafana/data'; +import { dateTime, observableTester, toUtc } from '@grafana/data'; import { PostgresDatasource } from '../datasource'; import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ import { TemplateSrv } from 'app/features/templating/template_srv'; import { initialCustomVariableModelState } from '../../../../features/variables/custom/reducer'; import { TimeSrv } from '../../../../features/dashboard/services/TimeSrv'; -import { observableTester } from '../../../../../test/helpers/observableTester'; jest.mock('@grafana/runtime', () => ({ ...((jest.requireActual('@grafana/runtime') as unknown) as object), diff --git a/public/app/plugins/panel/graph3/GraphPanel.tsx b/public/app/plugins/panel/graph3/GraphPanel.tsx index 2addf83b47d..d618d4eb29b 100644 --- a/public/app/plugins/panel/graph3/GraphPanel.tsx +++ b/public/app/plugins/panel/graph3/GraphPanel.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useEffect, useState } from 'react'; import { Area, Canvas, @@ -9,14 +9,15 @@ import { LegendPlugin, Line, Point, - SeriesGeometry, Scale, + SeriesGeometry, TooltipPlugin, UPlotChart, ZoomPlugin, } from '@grafana/ui'; import { + DataFrame, FieldConfig, FieldType, formattedValueToString, @@ -91,11 +92,18 @@ export const GraphPanel: React.FC = ({ options, onChangeTimeRange, }) => { - const alignedData = useMemo(() => { + const [alignedData, setAlignedData] = useState(null); + useEffect(() => { if (!data || !data.series?.length) { - return null; + setAlignedData(null); + return; } - return alignAndSortDataFramesByFieldName(data.series, TIME_FIELD_NAME); + + const subscription = alignAndSortDataFramesByFieldName(data.series, TIME_FIELD_NAME).subscribe(setAlignedData); + + return function unsubscribe() { + subscription.unsubscribe(); + }; }, [data]); if (!alignedData) { diff --git a/public/app/plugins/panel/graph3/utils.ts b/public/app/plugins/panel/graph3/utils.ts index af41232e73b..97b5bebec8a 100644 --- a/public/app/plugins/panel/graph3/utils.ts +++ b/public/app/plugins/panel/graph3/utils.ts @@ -1,7 +1,9 @@ +import { Observable } from 'rxjs'; import { DataFrame, FieldType, getTimeField, sortDataFrame, transformDataFrame } from '@grafana/data'; +import { map } from 'rxjs/operators'; // very time oriented for now -export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: string) => { +export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: string): Observable => { // normalize time field names // in each frame find first time field and rename it to unified name for (let i = 0; i < data.length; i++) { @@ -24,7 +26,7 @@ export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: // uPlot data needs to be aligned on x-axis (ref. https://github.com/leeoniya/uPlot/issues/108) // For experimentation just assuming alignment on time field, needs to change - const aligned = transformDataFrame( + return transformDataFrame( [ { id: 'seriesToColumns', @@ -32,8 +34,11 @@ export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: }, ], dataFramesToPlot - )[0]; - - // need to be more "clever", not time only in the future! - return sortDataFrame(aligned, getTimeField(aligned).timeIndex!); + ).pipe( + map(data => { + const aligned = data[0]; + // need to be more "clever", not time only in the future! + return sortDataFrame(aligned, getTimeField(aligned).timeIndex!); + }) + ); }; diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index 92ecf1456a5..9c5497ae400 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -1,20 +1,20 @@ import { Unsubscribable } from 'rxjs'; import { - HistoryItem, - DataQuery, - DataSourceApi, - QueryHint, - PanelData, - DataQueryRequest, - RawTimeRange, - LogLevel, - TimeRange, - LogsModel, - LogsDedupStrategy, AbsoluteTimeRange, - GraphSeriesXY, DataFrame, + DataQuery, + DataQueryRequest, + DataSourceApi, ExploreUrlState, + GraphSeriesXY, + HistoryItem, + LogLevel, + LogsDedupStrategy, + LogsModel, + PanelData, + QueryHint, + RawTimeRange, + TimeRange, } from '@grafana/data'; import { Emitter } from 'app/core/core'; @@ -219,3 +219,13 @@ export type RichHistoryQuery = { sessionName: string; timeRange?: string; }; + +export interface ExplorePanelData extends PanelData { + graphFrames: DataFrame[]; + tableFrames: DataFrame[]; + logsFrames: DataFrame[]; + traceFrames: DataFrame[]; + graphResult: GraphSeriesXY[] | null; + tableResult: DataFrame | null; + logsResult: LogsModel | null; +}