From 32054502100a5621ad28e35f4114ede18de24109 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 11 Aug 2021 01:25:33 -0700 Subject: [PATCH] Transformations: support a synchronous transformation pattern (#37780) --- .../transformers/ensureColumns.ts | 35 +++++++-------- .../transformations/transformers/histogram.ts | 33 ++++++-------- .../transformers/seriesToColumns.ts | 36 ++++++++------- .../grafana-data/src/types/transformations.ts | 15 +++++++ .../prepareTimeSeries.test.ts | 14 +++--- .../prepareTimeSeries/prepareTimeSeries.ts | 45 +++++++++---------- 6 files changed, 93 insertions(+), 85 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts index b8068ce3094..5b7f20511de 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts @@ -1,33 +1,28 @@ -import { of } from 'rxjs'; - import { seriesToColumnsTransformer } from './seriesToColumns'; import { DataFrame } from '../../types/dataFrame'; import { getTimeField } from '../../dataframe/processDataFrame'; -import { DataTransformerInfo } from '../../types/transformations'; +import { SynchronousDataTransformerInfo } from '../../types/transformations'; import { DataTransformerID } from './ids'; -import { mergeMap } from 'rxjs/operators'; +import { map } from 'rxjs/operators'; -export const ensureColumnsTransformer: DataTransformerInfo = { +export const ensureColumnsTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.ensureColumns, name: 'Ensure Columns Transformer', description: 'Will check if current data frames is series or columns. If in series it will convert to columns.', - operator: (options = {}) => (source) => - source.pipe( - mergeMap((data) => { - // Assume timeseries should first be joined by time - const timeFieldName = findConsistentTimeFieldName(data); - if (data.length > 1 && timeFieldName) { - return of(data).pipe( - seriesToColumnsTransformer.operator({ - byField: timeFieldName, - }) - ); - } + operator: (options) => (source) => source.pipe(map((data) => ensureColumnsTransformer.transformer(options)(data))), - return of(data); - }) - ), + transformer: (options: any) => (frames: DataFrame[]) => { + // Assume timeseries should first be joined by time + const timeFieldName = findConsistentTimeFieldName(frames); + + if (frames.length > 1 && timeFieldName) { + return seriesToColumnsTransformer.transformer({ + byField: timeFieldName, + })(frames); + } + return frames; + }, }; /** diff --git a/packages/grafana-data/src/transformations/transformers/histogram.ts b/packages/grafana-data/src/transformations/transformers/histogram.ts index a28aeaf88f9..0e8b2c1da88 100644 --- a/packages/grafana-data/src/transformations/transformers/histogram.ts +++ b/packages/grafana-data/src/transformations/transformers/histogram.ts @@ -1,4 +1,4 @@ -import { DataTransformerInfo } from '../../types'; +import { SynchronousDataTransformerInfo } from '../../types'; import { map } from 'rxjs/operators'; import { DataTransformerID } from './ids'; @@ -73,7 +73,7 @@ export const histogramFieldInfo = { /** * @alpha */ -export const histogramTransformer: DataTransformerInfo = { +export const histogramTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.histogram, name: 'Histogram', description: 'Calculate a histogram from input data', @@ -81,23 +81,18 @@ export const histogramTransformer: DataTransformerInfo (source) => - source.pipe( - map((data) => { - if (!Array.isArray(data) || data.length === 0) { - return data; - } - const hist = buildHistogram(data, options); - if (hist == null) { - return []; - } - return [histogramFieldsToFrame(hist)]; - }) - ), + operator: (options) => (source) => source.pipe(map((data) => histogramTransformer.transformer(options)(data))), + + transformer: (options: HistogramTransformerOptions) => (data: DataFrame[]) => { + if (!Array.isArray(data) || data.length === 0) { + return data; + } + const hist = buildHistogram(data, options); + if (hist == null) { + return []; + } + return [histogramFieldsToFrame(hist)]; + }, }; /** diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts index 07acaf87a2f..08170d1192b 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts @@ -1,6 +1,6 @@ import { map } from 'rxjs/operators'; -import { DataTransformerInfo, FieldMatcher } from '../../types'; +import { DataFrame, SynchronousDataTransformerInfo, FieldMatcher } from '../../types'; import { DataTransformerID } from './ids'; import { outerJoinDataFrames } from './joinDataFrames'; import { fieldMatchers } from '../matchers'; @@ -10,27 +10,29 @@ export interface SeriesToColumnsOptions { byField?: string; // empty will pick the field automatically } -export const seriesToColumnsTransformer: DataTransformerInfo = { +export const seriesToColumnsTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.seriesToColumns, name: 'Series as columns', // Called 'Outer join' in the UI! description: 'Groups series by field and returns values as columns', defaultOptions: { byField: undefined, // DEFAULT_KEY_FIELD, }, - operator: (options) => (source) => - source.pipe( - map((data) => { - if (data.length > 1) { - let joinBy: FieldMatcher | undefined = undefined; - if (options.byField) { - joinBy = fieldMatchers.get(FieldMatcherID.byName).get(options.byField); - } - const joined = outerJoinDataFrames({ frames: data, joinBy }); - if (joined) { - return [joined]; - } + + operator: (options) => (source) => source.pipe(map((data) => seriesToColumnsTransformer.transformer(options)(data))), + + transformer: (options: SeriesToColumnsOptions) => { + let joinBy: FieldMatcher | undefined = undefined; + return (data: DataFrame[]) => { + if (data.length > 1) { + if (options.byField && !joinBy) { + joinBy = fieldMatchers.get(FieldMatcherID.byName).get(options.byField); } - return data; - }) - ), + const joined = outerJoinDataFrames({ frames: data, joinBy }); + if (joined) { + return [joined]; + } + } + return data; + }; + }, }; diff --git a/packages/grafana-data/src/types/transformations.ts b/packages/grafana-data/src/types/transformations.ts index c1ad5995f61..b7a018093d7 100644 --- a/packages/grafana-data/src/types/transformations.ts +++ b/packages/grafana-data/src/types/transformations.ts @@ -5,6 +5,8 @@ import { RegistryItemWithOptions } from '../utils/Registry'; /** * Function that transform data frames (AKA transformer) + * + * @public */ export interface DataTransformerInfo extends RegistryItemWithOptions { /** @@ -14,6 +16,19 @@ export interface DataTransformerInfo extends RegistryItemWithOpt operator: (options: TOptions) => MonoTypeOperatorFunction; } +/** + * Many transformations can be called with a simple synchronous function. + * When a transformer is defined, it should have identical behavior to using the operator + * + * @public + */ +export interface SynchronousDataTransformerInfo extends DataTransformerInfo { + transformer: (options: TOptions) => (frames: DataFrame[]) => DataFrame[]; +} + +/** + * @public + */ export interface DataTransformerConfig { /** * Unique identifier of transformer diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts index cc043766fd4..b9cae546f84 100644 --- a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts +++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts @@ -1,5 +1,5 @@ import { toDataFrame, ArrayVector, DataFrame, FieldType, toDataFrameDTO, DataFrameDTO } from '@grafana/data'; -import { prepareTimeSeries, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries'; +import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries'; describe('Prepair time series transformer', () => { it('should transform wide to many', () => { @@ -19,7 +19,7 @@ describe('Prepair time series transformer', () => { format: timeSeriesFormat.TimeSeriesMany, }; - expect(prepareTimeSeries(source, config)).toEqual([ + expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([ toEquableDataFrame({ name: 'wide', refId: 'A', @@ -59,7 +59,7 @@ describe('Prepair time series transformer', () => { format: timeSeriesFormat.TimeSeriesMany, }; - expect(prepareTimeSeries(source, config)).toEqual([ + expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([ toEquableDataFrame({ name: 'wide', refId: 'A', @@ -107,7 +107,7 @@ describe('Prepair time series transformer', () => { format: timeSeriesFormat.TimeSeriesMany, }; - expect(prepareTimeSeries(source, config)).toEqual([ + expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([ toEquableDataFrame({ name: 'wide', refId: 'A', @@ -162,7 +162,9 @@ describe('Prepair time series transformer', () => { format: timeSeriesFormat.TimeSeriesMany, }; - expect(toEquableDataFrames(prepareTimeSeries(source, config))).toEqual(toEquableDataFrames(source)); + expect(toEquableDataFrames(prepareTimeSeriesTransformer.transformer(config)(source))).toEqual( + toEquableDataFrames(source) + ); }); it('should return empty array when no timeseries exist', () => { @@ -191,7 +193,7 @@ describe('Prepair time series transformer', () => { format: timeSeriesFormat.TimeSeriesMany, }; - expect(prepareTimeSeries(source, config)).toEqual([]); + expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([]); }); }); diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts index 197dc77c131..842c16cf5f6 100644 --- a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts +++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts @@ -1,5 +1,5 @@ import { - DataTransformerInfo, + SynchronousDataTransformerInfo, DataFrame, FieldType, DataTransformerID, @@ -63,31 +63,30 @@ export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] { return result; } -export function prepareTimeSeries(data: DataFrame[], options: PrepareTimeSeriesOptions): DataFrame[] { - const format = options?.format ?? timeSeriesFormat.TimeSeriesWide; - if (format === timeSeriesFormat.TimeSeriesMany) { - return toTimeSeriesMany(data); - } - - // Join by the first frame - const frame = outerJoinDataFrames({ - frames: data, - joinBy: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}), - enforceSort: true, - keepOriginIndices: true, - }); - return frame ? [frame] : []; -} - -export const prepareTimeSeriesTransformer: DataTransformerInfo = { +export const prepareTimeSeriesTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.prepareTimeSeries, name: 'Prepare time series', description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`, defaultOptions: {}, - /** - * Return a modified copy of the series. If the transform is not or should not - * be applied, just return the input series - */ - operator: (options) => (source) => source.pipe(map((data) => prepareTimeSeries(data, options))), + operator: (options) => (source) => + source.pipe(map((data) => prepareTimeSeriesTransformer.transformer(options)(data))), + + transformer: (options: PrepareTimeSeriesOptions) => { + const format = options?.format ?? timeSeriesFormat.TimeSeriesWide; + if (format === timeSeriesFormat.TimeSeriesMany) { + return toTimeSeriesMany; + } + + return (data: DataFrame[]) => { + // Join by the first frame + const frame = outerJoinDataFrames({ + frames: data, + joinBy: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}), + enforceSort: true, + keepOriginIndices: true, + }); + return frame ? [frame] : []; + }; + }, };