diff --git a/docs/sources/panels/transformations.md b/docs/sources/panels/transformations.md index bc9eb94640a..9dc06db1634 100644 --- a/docs/sources/panels/transformations.md +++ b/docs/sources/panels/transformations.md @@ -11,7 +11,7 @@ weight = 300 This page explains what transformations in Grafana are and how to use them. -> **Note:** This documentation refers to a Grafana 7.0 beta feature. This documentation will be frequently updated to reflect updates to the feature, and it will probably be broken into smaller sections when the feature moves out of beta. +> **Note:** This documentation refers to a Grafana 7.0 feature. This documentation will be frequently updated to reflect updates to the feature, and it will probably be broken into smaller sections when the feature moves out of beta. Transformations process the result set before it’s passed to the visualization. You access transformations in the Transform tab of the Grafana panel editor. @@ -74,6 +74,7 @@ Grafana comes with the following transformations: - [Join by field (outer join)](#join-by-field-outer-join) - [Add field from calculation](#add-field-from-calculation) - [Labels to fields](#labels-to-fields) + - [Series to rows](#series-to-rows) - [Debug transformations](#debug-transformations) Keep reading for detailed descriptions of each type of transformation and the options available for each, as well as suggestions on how to use them. @@ -96,25 +97,33 @@ After I apply the transformation, there is no time value and each column has bee ### Merge -Use this transformation to combine the result from multiple queries into one single result based on the time field. This is helpful when using the table panel visualization. +> **Note:** This documentation refers to a Grafana 7.1 feature. -In the example below, we are visualizing multiple queries returning table data before applying the transformation. +Use this transformation to combine the result from multiple queries into one single result. This is helpful when using the table panel visualization. Values that can be merged are combined into the same row. Values are mergeable if the shared fields contains the same data. -{{< docs-imagebox img="/img/docs/transformations/table-data-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} +In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the transformation. -Here is the same example after applying the merge transformation. +Query A: -{{< docs-imagebox img="/img/docs/transformations/table-data-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} +| Time | Job | Uptime | +|---------------------|---------|-----------| +| 2020-07-07 11:34:20 | node | 25260122 | +| 2020-07-07 11:24:20 | postgre | 123001233 | -If any of the queries return time series data, then a `Metric` column containing the name of the query is added. You can be customized this value by defining `Label` on the source query. +Query B: -In the example below, we are visualizing multiple queries returning time series data before applying the transformation. +| Time | Job | Errors | +|---------------------|---------|--------| +| 2020-07-07 11:34:20 | node | 15 | +| 2020-07-07 11:24:20 | postgre | 5 | -{{< docs-imagebox img="/img/docs/transformations/time-series-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} +Here is the result after applying the `Merge` transformation. -Here is the same example after applying the merge transformation. +| Time | Job | Errors | Uptime | +|---------------------|---------|--------|-----------| +| 2020-07-07 11:34:20 | node | 15 | 25260122 | +| 2020-07-07 11:24:20 | postgre | 5 | 123001233 | -{{< docs-imagebox img="/img/docs/transformations/time-series-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} ### Filter by name @@ -213,6 +222,43 @@ After I apply the transformation, my labels appear in the table as fields. {{< docs-imagebox img="/img/docs/transformations/labels-to-fields-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} +## Series to rows + +> **Note:** This documentation refers to a Grafana 7.1 feature. + +Use this transformation to combine the result from multiple time series data queries into one single result. This is helpful when using the table panel visualization. + +The result from this transformation will contain three columns: `Time`, `Metric`, and `Value`. The `Metric` column is added so you easily can see from which query the metric originates from. Customize this value by defining `Label` on the source query. + +In the example below, we have two queries returning time series data. It is visualized as two separate tables before applying the transformation. + +Query A: + +| Time | Temperature | +|---------------------|-------------| +| 2020-07-07 11:34:20 | 25 | +| 2020-07-07 10:31:22 | 22 | +| 2020-07-07 09:30:05 | 19 | + +Query B: + +| Time | Humidity | +|---------------------|----------| +| 2020-07-07 11:34:20 | 24 | +| 2020-07-07 10:32:20 | 29 | +| 2020-07-07 09:30:57 | 33 | + +Here is the result after applying the `Series to rows` transformation. + +| Time | Metric | Value | +|---------------------|-------------|-------| +| 2020-07-07 11:34:20 | Temperature | 25 | +| 2020-07-07 11:34:20 | Humidity | 22 | +| 2020-07-07 10:32:20 | Humidity | 29 | +| 2020-07-07 10:31:22 | Temperature | 22 | +| 2020-07-07 09:30:57 | Humidity | 33 | +| 2020-07-07 09:30:05 | Temperature | 19 | + ## Debug transformations To see the input and the output result sets of the transformation, click the bug icon on the right side of the transformation row. diff --git a/packages/grafana-data/src/dataframe/utils.ts b/packages/grafana-data/src/dataframe/utils.ts new file mode 100644 index 00000000000..094d8fbeffd --- /dev/null +++ b/packages/grafana-data/src/dataframe/utils.ts @@ -0,0 +1,12 @@ +import { DataFrame, FieldType } from '../types/dataFrame'; + +export const isTimeSerie = (frame: DataFrame): boolean => { + if (frame.fields.length > 2) { + return false; + } + return !!frame.fields.find(field => field.type === FieldType.time); +}; + +export const isTimeSeries = (data: DataFrame[]): boolean => { + return !data.find(frame => !isTimeSerie(frame)); +}; diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index 636fbbe130e..9b4df802f85 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -8,10 +8,11 @@ import { filterFramesByRefIdTransformer } from './transformers/filterByRefId'; import { orderFieldsTransformer } from './transformers/order'; import { organizeFieldsTransformer } from './transformers/organize'; import { seriesToColumnsTransformer } from './transformers/seriesToColumns'; +import { seriesToRowsTransformer } from './transformers/seriesToRows'; import { renameFieldsTransformer } from './transformers/rename'; import { labelsToFieldsTransformer } from './transformers/labelsToFields'; import { ensureColumnsTransformer } from './transformers/ensureColumns'; -import { mergeTransformer } from './transformers/merge/merge'; +import { mergeTransformer } from './transformers/merge'; export const standardTransformers = { noopTransformer, @@ -25,6 +26,7 @@ export const standardTransformers = { reduceTransformer, calculateFieldTransformer, seriesToColumnsTransformer, + seriesToRowsTransformer, renameFieldsTransformer, labelsToFieldsTransformer, ensureColumnsTransformer, diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index d12caf2710e..2c619b2809f 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -8,6 +8,7 @@ export enum DataTransformerID { rename = 'rename', calculateField = 'calculateField', seriesToColumns = 'seriesToColumns', + seriesToRows = 'seriesToRows', merge = 'merge', labelsToFields = 'labelsToFields', filterFields = 'filterFields', diff --git a/packages/grafana-data/src/transformations/transformers/merge/merge.test.ts b/packages/grafana-data/src/transformations/transformers/merge.test.ts similarity index 74% rename from packages/grafana-data/src/transformations/transformers/merge/merge.test.ts rename to packages/grafana-data/src/transformations/transformers/merge.test.ts index e27ac109d44..171eb5cc992 100644 --- a/packages/grafana-data/src/transformations/transformers/merge/merge.test.ts +++ b/packages/grafana-data/src/transformations/transformers/merge.test.ts @@ -1,9 +1,9 @@ -import { mockTransformationsRegistry } from '../../../utils/tests/mockTransformationsRegistry'; -import { DataTransformerConfig, Field, FieldType } from '../../../types'; -import { DataTransformerID } from '../ids'; -import { toDataFrame } from '../../../dataframe'; -import { transformDataFrame } from '../../transformDataFrame'; -import { ArrayVector } from '../../../vector'; +import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { DataTransformerConfig, Field, FieldType } from '../../types'; +import { DataTransformerID } from './ids'; +import { toDataFrame } from '../../dataframe'; +import { transformDataFrame } from '../transformDataFrame'; +import { ArrayVector } from '../../vector'; import { mergeTransformer, MergeTransformerOptions } from './merge'; describe('Merge multipe to single', () => { @@ -35,12 +35,11 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [seriesA, seriesB]); const expected: Field[] = [ - createField('Time', FieldType.time, [1000, 2000]), - createField('Metric', FieldType.string, ['A', 'B']), - createField('Value', FieldType.number, [1, -1]), + createField('Time', FieldType.time, [2000, 1000]), + createField('Temp', FieldType.number, [-1, 1]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine two series with multiple values into one', () => { @@ -67,12 +66,11 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [seriesA, seriesB]); const expected: Field[] = [ - createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), - createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), - createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine three series into one', () => { @@ -107,12 +105,11 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]); const expected: Field[] = [ - createField('Time', FieldType.time, [500, 1000, 2000]), - createField('Metric', FieldType.string, ['C', 'A', 'B']), - createField('Value', FieldType.number, [2, 1, -1]), + createField('Time', FieldType.time, [2000, 1000, 500]), + createField('Temp', FieldType.number, [-1, 1, 2]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine one serie and two tables into one table', () => { @@ -149,13 +146,12 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [tableA, seriesB, tableB]); const expected: Field[] = [ - createField('Time', FieldType.time, [500, 1000, 1000]), - createField('Metric', FieldType.string, ['C', 'A', 'B']), - createField('Temp', FieldType.number, [2, 1, -1]), - createField('Humidity', FieldType.number, [5, 10, null]), + createField('Time', FieldType.time, [1000, 1000, 500]), + createField('Temp', FieldType.number, [1, -1, 2]), + createField('Humidity', FieldType.number, [10, null, 5]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine one serie and two tables with ISO dates into one table', () => { @@ -192,13 +188,12 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [tableA, seriesB, tableC]); const expected: Field[] = [ - createField('Time', FieldType.time, ['2019-09-01T11:10:23Z', '2019-10-01T11:10:23Z', '2019-11-01T11:10:23Z']), - createField('Metric', FieldType.string, ['B', 'A', 'C']), - createField('Temp', FieldType.number, [-1, 1, 2]), - createField('Humidity', FieldType.number, [null, 10, 5]), + createField('Time', FieldType.time, ['2019-11-01T11:10:23Z', '2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z']), + createField('Temp', FieldType.number, [2, 1, -1]), + createField('Humidity', FieldType.number, [5, 10, null]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine three tables with multiple values into one', () => { @@ -235,14 +230,15 @@ describe('Merge multipe to single', () => { }); const result = transformDataFrame([cfg], [tableA, tableB, tableC]); + const expected: Field[] = [ - createField('Time', FieldType.time, [100, 100, 100, 124, 125, 126, 149, 150, 200]), - createField('Temp', FieldType.number, [1, -1, 1, 4, 2, 3, 5, 4, 5]), - createField('Humidity', FieldType.number, [10, null, 22, 25, null, null, 30, 14, 55]), - createField('Enabled', FieldType.boolean, [null, true, null, null, false, true, null, null, null]), + createField('Time', FieldType.time, [200, 150, 149, 126, 125, 124, 100, 100, 100]), + createField('Temp', FieldType.number, [5, 4, 5, 3, 2, 4, 1, -1, 1]), + createField('Humidity', FieldType.number, [55, 14, 30, null, null, 25, 10, null, 22]), + createField('Enabled', FieldType.boolean, [null, null, null, true, false, null, null, true, null]), ]; - expect(result[0].fields).toMatchObject(expected); + expect(unwrap(result[0].fields)).toEqual(expected); }); it('combine two time series, where first serie fields has displayName, into one', () => { @@ -269,13 +265,14 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [serieA, serieB]); const expected: Field[] = [ - createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), - createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), - createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]), ]; - expect(result[0].fields[2].config).toEqual({}); - expect(result[0].fields).toMatchObject(expected); + const fields = unwrap(result[0].fields); + + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); }); it('combine two time series, where first serie fields has units, into one', () => { @@ -302,13 +299,14 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [serieA, serieB]); const expected: Field[] = [ - createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), - createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), - createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5], { units: 'celsius' }), + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }), ]; - expect(result[0].fields[2].config).toEqual({ units: 'celsius' }); - expect(result[0].fields).toMatchObject(expected); + const fields = unwrap(result[0].fields); + + expect(fields[1].config).toEqual({ units: 'celsius' }); + expect(fields).toEqual(expected); }); it('combine two time series, where second serie fields has units, into one', () => { @@ -335,16 +333,28 @@ describe('Merge multipe to single', () => { const result = transformDataFrame([cfg], [serieA, serieB]); const expected: Field[] = [ - createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), - createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), - createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]), ]; - expect(result[0].fields[2].config).toEqual({}); - expect(result[0].fields).toMatchObject(expected); + const fields = unwrap(result[0].fields); + + expect(fields[1].config).toEqual({}); + expect(fields).toEqual(expected); }); }); const createField = (name: string, type: FieldType, values: any[], config = {}): Field => { return { name, type, values: new ArrayVector(values), config, labels: undefined }; }; + +const unwrap = (fields: Field[]): Field[] => { + return fields.map(field => + createField( + field.name, + field.type, + field.values.toArray().map((value: any) => value), + field.config + ) + ); +}; diff --git a/packages/grafana-data/src/transformations/transformers/merge.ts b/packages/grafana-data/src/transformations/transformers/merge.ts new file mode 100644 index 00000000000..aa623d6601d --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge.ts @@ -0,0 +1,216 @@ +import { DataTransformerID } from './ids'; +import { DataTransformerInfo } from '../../types/transformations'; +import { DataFrame, Field, FieldType } from '../../types/dataFrame'; +import { omit } from 'lodash'; +import { ArrayVector } from '../../vector/ArrayVector'; +import { MutableDataFrame, sortDataFrame } from '../../dataframe'; + +type MergeDetailsKeyFactory = (existing: Record, value: Record) => string; + +export interface MergeTransformerOptions {} + +export const mergeTransformer: DataTransformerInfo = { + id: DataTransformerID.merge, + name: 'Merge series/tables', + description: 'Merges multiple series/tables into a single serie/table', + defaultOptions: {}, + transformer: (options: MergeTransformerOptions) => { + return (data: DataFrame[]) => { + if (!Array.isArray(data) || data.length <= 1) { + return data; + } + + const fieldByName = new Set(); + const fieldIndexByName: Record> = {}; + const fieldNamesForKey: string[] = []; + const dataFrame = new MutableDataFrame(); + + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const field = frame.fields[fieldIndex]; + + if (!fieldByName.has(field.name)) { + dataFrame.addField(copyFieldStructure(field)); + fieldByName.add(field.name); + } + + fieldIndexByName[field.name] = fieldIndexByName[field.name] || {}; + fieldIndexByName[field.name][frameIndex] = fieldIndex; + + if (data.length - 1 !== frameIndex) { + continue; + } + + if (Object.keys(fieldIndexByName[field.name]).length === data.length) { + fieldNamesForKey.push(field.name); + } + } + } + + if (fieldNamesForKey.length === 0) { + return data; + } + + const dataFrameIndexByKey: Record = {}; + const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey); + const detailsKeyFactory = createDetailsKeyFactory(fieldByName, fieldNamesForKey); + const valueMapper = createValueMapper(data, fieldByName, fieldIndexByName); + + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + + for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { + const key = keyFactory(frameIndex, valueIndex); + const value = valueMapper(frameIndex, valueIndex); + mergeOrAdd(key, value, dataFrame, dataFrameIndexByKey, detailsKeyFactory); + } + } + + const timeIndex = dataFrame.fields.findIndex(field => field.type === FieldType.time); + if (typeof timeIndex === 'number') { + return [sortDataFrame(dataFrame, timeIndex, true)]; + } + return [dataFrame]; + }; + }, +}; + +const copyFieldStructure = (field: Field): Field => { + return { + ...omit(field, ['values', 'state', 'labels', 'config']), + values: new ArrayVector(), + config: { + ...omit(field.config, 'displayName'), + }, + }; +}; + +const createKeyFactory = ( + data: DataFrame[], + fieldPointerByName: Record>, + keyFieldNames: string[] +) => { + const factoryIndex = keyFieldNames.reduce((index: Record, fieldName) => { + return Object.keys(fieldPointerByName[fieldName]).reduce((index: Record, frameIndex) => { + index[frameIndex] = index[frameIndex] || []; + index[frameIndex].push(fieldPointerByName[fieldName][frameIndex]); + return index; + }, index); + }, {}); + + return (frameIndex: number, valueIndex: number): string => { + return factoryIndex[frameIndex].reduce((key: string, fieldIndex: number) => { + return key + data[frameIndex].fields[fieldIndex].values.get(valueIndex); + }, ''); + }; +}; + +const createDetailsKeyFactory = (fieldByName: Set, fieldNamesForKey: string[]): MergeDetailsKeyFactory => { + const fieldNamesToExclude = fieldNamesForKey.reduce((exclude: Record, fieldName: string) => { + exclude[fieldName] = true; + return exclude; + }, {}); + + const checkOrder = Array.from(fieldByName).filter(fieldName => !fieldNamesToExclude[fieldName]); + + return (existing: Record, value: Record) => { + return checkOrder.reduce((key: string, fieldName: string) => { + if (typeof existing[fieldName] === 'undefined') { + return key; + } + if (typeof value[fieldName] === 'undefined') { + return key; + } + if (existing[fieldName] === value[fieldName]) { + return key; + } + return key + value[fieldName]; + }, ''); + }; +}; + +const createValueMapper = ( + data: DataFrame[], + fieldByName: Set, + fieldIndexByName: Record> +) => { + return (frameIndex: number, valueIndex: number) => { + const value: Record = {}; + const fieldNames = Array.from(fieldByName); + + for (const fieldName of fieldNames) { + const fieldIndexByFrameIndex = fieldIndexByName[fieldName]; + if (!fieldIndexByFrameIndex) { + continue; + } + + const fieldIndex = fieldIndexByFrameIndex[frameIndex]; + if (typeof fieldIndex !== 'number') { + continue; + } + + const frame = data[frameIndex]; + if (!frame || !frame.fields) { + continue; + } + + const field = frame.fields[fieldIndex]; + if (!field || !field.values) { + continue; + } + + value[fieldName] = field.values.get(valueIndex); + } + + return value; + }; +}; + +const isMergable = (existing: Record, value: Record): boolean => { + let mergable = true; + + for (const prop in value) { + if (typeof existing[prop] === 'undefined') { + continue; + } + + if (existing[prop] === null) { + continue; + } + + if (existing[prop] !== value[prop]) { + mergable = false; + break; + } + } + + return mergable; +}; + +const mergeOrAdd = ( + key: string, + value: Record, + dataFrame: MutableDataFrame, + dataFrameIndexByKey: Record, + detailsKeyFactory: MergeDetailsKeyFactory +) => { + if (typeof dataFrameIndexByKey[key] === 'undefined') { + dataFrame.add(value); + dataFrameIndexByKey[key] = dataFrame.length - 1; + return; + } + + const dataFrameIndex = dataFrameIndexByKey[key]; + const existing = dataFrame.get(dataFrameIndex); + + if (isMergable(existing, value)) { + const merged = { ...existing, ...value }; + dataFrame.set(dataFrameIndex, merged); + return; + } + + const nextKey = key + detailsKeyFactory(existing, value); + mergeOrAdd(nextKey, value, dataFrame, dataFrameIndexByKey, detailsKeyFactory); +}; diff --git a/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts b/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts deleted file mode 100644 index 9f36435b1e1..00000000000 --- a/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { MutableDataFrame } from '../../../dataframe'; -import { - DataFrame, - FieldType, - Field, - TIME_SERIES_TIME_FIELD_NAME, - TIME_SERIES_VALUE_FIELD_NAME, -} from '../../../types/dataFrame'; -import { ArrayVector } from '../../../vector'; -import { omit } from 'lodash'; -import { getFrameDisplayName } from '../../../field'; - -interface DataFrameBuilderResult { - dataFrame: MutableDataFrame; - valueMapper: ValueMapper; -} - -type ValueMapper = (frame: DataFrame, valueIndex: number, timeIndex: number) => Record; - -const TIME_SERIES_METRIC_FIELD_NAME = 'Metric'; - -export class DataFrameBuilder { - private isOnlyTimeSeries: boolean; - private displayMetricField: boolean; - private valueFields: Record; - private timeField: Field | null; - - constructor() { - this.isOnlyTimeSeries = true; - this.displayMetricField = false; - this.valueFields = {}; - this.timeField = null; - } - - addFields(frame: DataFrame, timeIndex: number): void { - if (frame.fields.length > 2) { - this.isOnlyTimeSeries = false; - } - - if (frame.fields.length === 2) { - this.displayMetricField = true; - } - - for (let index = 0; index < frame.fields.length; index++) { - const field = frame.fields[index]; - - if (index === timeIndex) { - if (!this.timeField) { - this.timeField = this.copyStructure(field, TIME_SERIES_TIME_FIELD_NAME); - } - continue; - } - - if (!this.valueFields[field.name]) { - this.valueFields[field.name] = this.copyStructure(field, field.name); - } - } - } - - build(): DataFrameBuilderResult { - return { - dataFrame: this.createDataFrame(), - valueMapper: this.createValueMapper(), - }; - } - - private createValueMapper(): ValueMapper { - return (frame: DataFrame, valueIndex: number, timeIndex: number) => { - return frame.fields.reduce((values: Record, field, index) => { - const value = field.values.get(valueIndex); - - if (index === timeIndex) { - values[TIME_SERIES_TIME_FIELD_NAME] = value; - - if (this.displayMetricField) { - values[TIME_SERIES_METRIC_FIELD_NAME] = getFrameDisplayName(frame); - } - return values; - } - - if (this.isOnlyTimeSeries) { - values[TIME_SERIES_VALUE_FIELD_NAME] = value; - return values; - } - - values[field.name] = value; - return values; - }, {}); - }; - } - - private createDataFrame(): MutableDataFrame { - const dataFrame = new MutableDataFrame(); - - if (this.timeField) { - dataFrame.addField(this.timeField); - - if (this.displayMetricField) { - dataFrame.addField({ - name: TIME_SERIES_METRIC_FIELD_NAME, - type: FieldType.string, - }); - } - } - - const valueFields = Object.values(this.valueFields); - - if (this.isOnlyTimeSeries) { - if (valueFields.length > 0) { - dataFrame.addField({ - ...valueFields[0], - name: TIME_SERIES_VALUE_FIELD_NAME, - }); - } - return dataFrame; - } - - for (const field of valueFields) { - dataFrame.addField(field); - } - - return dataFrame; - } - - private copyStructure(field: Field, name: string): Field { - return { - ...omit(field, ['values', 'name', 'state', 'labels', 'config']), - name, - values: new ArrayVector(), - config: { - ...omit(field.config, 'displayName'), - }, - }; - } -} diff --git a/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts b/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts deleted file mode 100644 index 3d19acc605a..00000000000 --- a/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { DataFrame } from '../../../types/dataFrame'; -import { timeComparer } from '../../../field/fieldComparers'; -import { sortDataFrame } from '../../../dataframe'; -import { TimeFieldsByFrame } from './TimeFieldsByFrame'; - -interface DataFrameStackValue { - valueIndex: number; - timeIndex: number; - frame: DataFrame; -} -export class DataFramesStackedByTime { - private valuesPointerByFrame: Record; - private dataFrames: DataFrame[]; - private isSorted: boolean; - - constructor(private timeFields: TimeFieldsByFrame) { - this.valuesPointerByFrame = {}; - this.dataFrames = []; - this.isSorted = false; - } - - push(frame: DataFrame): number { - const index = this.dataFrames.length; - this.valuesPointerByFrame[index] = 0; - this.dataFrames.push(frame); - return index; - } - - pop(): DataFrameStackValue { - if (!this.isSorted) { - this.sortByTime(); - this.isSorted = true; - } - - const frameIndex = this.dataFrames.reduce((champion, frame, index) => { - const championTime = this.peekTimeValueForFrame(champion); - const contenderTime = this.peekTimeValueForFrame(index); - return timeComparer(contenderTime, championTime) >= 0 ? champion : index; - }, 0); - - const previousPointer = this.movePointerForward(frameIndex); - - return { - frame: this.dataFrames[frameIndex], - valueIndex: previousPointer, - timeIndex: this.timeFields.getFieldIndex(frameIndex), - }; - } - - getLength(): number { - const frames = Object.values(this.dataFrames); - return frames.reduce((length: number, frame) => (length += frame.length), 0); - } - - private peekTimeValueForFrame(frameIndex: number): any { - const timeField = this.timeFields.getField(frameIndex); - const valuePointer = this.valuesPointerByFrame[frameIndex]; - return timeField.values.get(valuePointer); - } - - private movePointerForward(frameIndex: number): number { - const currentPointer = this.valuesPointerByFrame[frameIndex]; - this.valuesPointerByFrame[frameIndex] = currentPointer + 1; - - return currentPointer; - } - - private sortByTime() { - this.dataFrames = this.dataFrames.map((frame, index) => { - const timeFieldIndex = this.timeFields.getFieldIndex(index); - return sortDataFrame(frame, timeFieldIndex); - }); - } -} diff --git a/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts b/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts deleted file mode 100644 index c1f44d6d516..00000000000 --- a/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { isNumber } from 'lodash'; -import { Field, DataFrame } from '../../../types/dataFrame'; -import { getTimeField } from '../../../dataframe'; - -export class TimeFieldsByFrame { - private timeIndexByFrameIndex: Record; - private timeFieldByFrameIndex: Record; - - constructor() { - this.timeIndexByFrameIndex = {}; - this.timeFieldByFrameIndex = {}; - } - - add(frameIndex: number, frame: DataFrame): void { - const fieldDescription = getTimeField(frame); - const timeIndex = fieldDescription?.timeIndex; - const timeField = fieldDescription?.timeField; - - if (isNumber(timeIndex)) { - this.timeIndexByFrameIndex[frameIndex] = timeIndex; - } - - if (timeField) { - this.timeFieldByFrameIndex[frameIndex] = timeField; - } - } - - getField(frameIndex: number): Field { - return this.timeFieldByFrameIndex[frameIndex]; - } - - getFieldIndex(frameIndex: number): number { - return this.timeIndexByFrameIndex[frameIndex]; - } - - getLength() { - return Object.keys(this.timeIndexByFrameIndex).length; - } -} diff --git a/packages/grafana-data/src/transformations/transformers/merge/merge.ts b/packages/grafana-data/src/transformations/transformers/merge/merge.ts deleted file mode 100644 index 15d7b70f99a..00000000000 --- a/packages/grafana-data/src/transformations/transformers/merge/merge.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { DataTransformerID } from '../ids'; -import { DataTransformerInfo } from '../../../types/transformations'; -import { DataFrame } from '../../../types/dataFrame'; -import { DataFrameBuilder } from './DataFrameBuilder'; -import { TimeFieldsByFrame } from './TimeFieldsByFrame'; -import { DataFramesStackedByTime } from './DataFramesStackedByTime'; - -export interface MergeTransformerOptions {} - -export const mergeTransformer: DataTransformerInfo = { - id: DataTransformerID.merge, - name: 'Merge series/tables', - description: 'Merges multiple series/tables by time into a single serie/table', - defaultOptions: {}, - transformer: (options: MergeTransformerOptions) => { - return (data: DataFrame[]) => { - if (!Array.isArray(data) || data.length <= 1) { - return data; - } - - const timeFields = new TimeFieldsByFrame(); - const framesStack = new DataFramesStackedByTime(timeFields); - const dataFrameBuilder = new DataFrameBuilder(); - - for (const frame of data) { - const frameIndex = framesStack.push(frame); - timeFields.add(frameIndex, frame); - - const timeIndex = timeFields.getFieldIndex(frameIndex); - dataFrameBuilder.addFields(frame, timeIndex); - } - - if (data.length !== timeFields.getLength()) { - return data; - } - - const { dataFrame, valueMapper } = dataFrameBuilder.build(); - - for (let index = 0; index < framesStack.getLength(); index++) { - const { frame, valueIndex, timeIndex } = framesStack.pop(); - dataFrame.add(valueMapper(frame, valueIndex, timeIndex)); - } - - return [dataFrame]; - }; - }, -}; diff --git a/packages/grafana-data/src/transformations/transformers/seriesToRows.test.ts b/packages/grafana-data/src/transformations/transformers/seriesToRows.test.ts new file mode 100644 index 00000000000..d89719985c6 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/seriesToRows.test.ts @@ -0,0 +1,237 @@ +import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { DataTransformerConfig, Field, FieldType } from '../../types'; +import { DataTransformerID } from './ids'; +import { toDataFrame } from '../../dataframe'; +import { transformDataFrame } from '../transformDataFrame'; +import { ArrayVector } from '../../vector'; +import { seriesToRowsTransformer, SeriesToRowsTransformerOptions } from './seriesToRows'; + +describe('Series to rows', () => { + beforeAll(() => { + mockTransformationsRegistry([seriesToRowsTransformer]); + }); + + it('combine two series into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [2000] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [2000, 1000]), + createField('Metric', FieldType.string, ['B', 'A']), + createField('Value', FieldType.number, [-1, 1]), + ]; + + expect(unwrap(result[0].fields)).toEqual(expected); + }); + + it('combine two series with multiple values into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; + + expect(unwrap(result[0].fields)).toEqual(expected); + }); + + it('combine three series into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [2000] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const seriesC = toDataFrame({ + name: 'C', + fields: [ + { name: 'Time', type: FieldType.time, values: [500] }, + { name: 'Temp', type: FieldType.number, values: [2] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]); + const expected: Field[] = [ + createField('Time', FieldType.time, [2000, 1000, 500]), + createField('Metric', FieldType.string, ['B', 'A', 'C']), + createField('Value', FieldType.number, [-1, 1, 2]), + ]; + + expect(unwrap(result[0].fields)).toEqual(expected); + }); + + it('combine two time series, where first serie fields has displayName, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200], config: { displayName: 'Random time' } }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { displayName: 'Temp' } }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; + + const fields = unwrap(result[0].fields); + + expect(fields[2].config).toEqual({}); + expect(fields).toEqual(expected); + }); + + it('combine two time series, where first serie fields has units, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { units: 'celsius' } }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }), + ]; + + const fields = unwrap(result[0].fields); + + expect(fields[2].config).toEqual({ units: 'celsius' }); + expect(fields).toEqual(expected); + }); + + it('combine two time series, where second serie fields has units, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToRows, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3], config: { units: 'celsius' } }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]), + createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']), + createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]), + ]; + + const fields = unwrap(result[0].fields); + + expect(fields[2].config).toEqual({}); + expect(fields).toEqual(expected); + }); +}); + +const createField = (name: string, type: FieldType, values: any[], config = {}): Field => { + return { name, type, values: new ArrayVector(values), config, labels: undefined }; +}; + +const unwrap = (fields: Field[]): Field[] => { + return fields.map(field => + createField( + field.name, + field.type, + field.values.toArray().map((value: any) => value), + field.config + ) + ); +}; diff --git a/packages/grafana-data/src/transformations/transformers/seriesToRows.ts b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts new file mode 100644 index 00000000000..ee2966a7ed5 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts @@ -0,0 +1,97 @@ +import { omit } from 'lodash'; +import { DataTransformerID } from './ids'; +import { DataTransformerInfo } from '../../types/transformations'; +import { + DataFrame, + Field, + FieldType, + TIME_SERIES_TIME_FIELD_NAME, + TIME_SERIES_VALUE_FIELD_NAME, + TIME_SERIES_METRIC_FIELD_NAME, +} from '../../types/dataFrame'; +import { isTimeSeries } from '../../dataframe/utils'; +import { MutableDataFrame, sortDataFrame } from '../../dataframe'; +import { ArrayVector } from '../../vector'; +import { getFrameDisplayName } from '../../field/fieldState'; + +export interface SeriesToRowsTransformerOptions {} + +export const seriesToRowsTransformer: DataTransformerInfo = { + id: DataTransformerID.seriesToRows, + name: 'Series to rows', + description: 'Combines multiple series into a single serie and appends a column with metric name per value.', + defaultOptions: {}, + transformer: (options: SeriesToRowsTransformerOptions) => { + return (data: DataFrame[]) => { + if (!Array.isArray(data) || data.length <= 1) { + return data; + } + + if (!isTimeSeries(data)) { + return data; + } + + const timeFieldByIndex: Record = {}; + const targetFields = new Set(); + const dataFrame = new MutableDataFrame(); + const metricField: Field = { + name: TIME_SERIES_METRIC_FIELD_NAME, + values: new ArrayVector(), + config: {}, + type: FieldType.string, + }; + + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const field = frame.fields[fieldIndex]; + + if (field.type === FieldType.time) { + timeFieldByIndex[frameIndex] = fieldIndex; + + if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) { + dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME)); + dataFrame.addField(metricField); + targetFields.add(TIME_SERIES_TIME_FIELD_NAME); + } + continue; + } + + if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) { + dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME)); + targetFields.add(TIME_SERIES_VALUE_FIELD_NAME); + } + } + } + + for (let frameIndex = 0; frameIndex < data.length; frameIndex++) { + const frame = data[frameIndex]; + + for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) { + const timeFieldIndex = timeFieldByIndex[frameIndex]; + const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0; + + dataFrame.add({ + [TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex), + [TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame), + [TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex), + }); + } + } + + return [sortDataFrame(dataFrame, 0, true)]; + }; + }, +}; + +const copyFieldStructure = (field: Field, name: string): Field => { + return { + ...omit(field, ['values', 'state', 'labels', 'config', 'name']), + name: name, + values: new ArrayVector(), + config: { + ...omit(field.config, 'displayName'), + }, + }; +}; diff --git a/packages/grafana-data/src/types/dataFrame.ts b/packages/grafana-data/src/types/dataFrame.ts index 05d54d69fa6..380cab80a00 100644 --- a/packages/grafana-data/src/types/dataFrame.ts +++ b/packages/grafana-data/src/types/dataFrame.ts @@ -150,3 +150,4 @@ export interface FieldCalcs extends Record {} export const TIME_SERIES_VALUE_FIELD_NAME = 'Value'; export const TIME_SERIES_TIME_FIELD_NAME = 'Time'; +export const TIME_SERIES_METRIC_FIELD_NAME = 'Metric'; diff --git a/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx b/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx index dfa4b723e5d..9886291503f 100644 --- a/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx +++ b/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { DataTransformerID, standardTransformers, TransformerRegistyItem, TransformerUIProps } from '@grafana/data'; -import { MergeTransformerOptions } from '@grafana/data/src/transformations/transformers/merge/merge'; +import { MergeTransformerOptions } from '@grafana/data/src/transformations/transformers/merge'; export const MergeTransformerEditor: React.FC> = ({ input, @@ -14,7 +14,7 @@ export const mergeTransformerRegistryItem: TransformerRegistyItem> = ({ + input, + options, + onChange, +}) => { + return null; +}; + +export const seriesToRowsTransformerRegistryItem: TransformerRegistyItem = { + id: DataTransformerID.seriesToRows, + editor: SeriesToRowsTransformerEditor, + transformation: standardTransformers.seriesToRowsTransformer, + name: 'Series to rows', + description: `Merge many series and return a single series with time, metric and value as columns. + Useful for showing multiple time series visualized in a table.`, +}; diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts index f7437d40624..9d353a2db0d 100644 --- a/public/app/core/utils/standardTransformers.ts +++ b/public/app/core/utils/standardTransformers.ts @@ -7,6 +7,7 @@ import { seriesToFieldsTransformerRegistryItem } from '../components/Transformer import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor'; import { labelsToFieldsTransformerRegistryItem } from '../components/TransformersUI/LabelsToFieldsTransformerEditor'; import { mergeTransformerRegistryItem } from '../components/TransformersUI/MergeTransformerEditor'; +import { seriesToRowsTransformerRegistryItem } from '../components/TransformersUI/SeriesToRowsTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -15,6 +16,7 @@ export const getStandardTransformers = (): Array> => filterFramesByRefIdTransformRegistryItem, organizeFieldsTransformRegistryItem, seriesToFieldsTransformerRegistryItem, + seriesToRowsTransformerRegistryItem, calculateFieldTransformRegistryItem, labelsToFieldsTransformerRegistryItem, mergeTransformerRegistryItem,