Transform: adding missing "table"-transform and "series to rows"-transform to Grafana v7-transforms. (#26042)

* Fixed so the merge for table values works as it did before.

* wip

* fixed tests.

* merge tests are green.

* removed unused code and simplify the seriesToRows.

* added time series to rows editor.

* using getFrameDisplayName for the metric value.

* updated description of transforms.

* updated docs.

* fixed according to feedback.

* changed from images to markdown tables for the examples.

* forgot to save :P

(cherry picked from commit 17d87071e6)
This commit is contained in:
Marcus Andersson
2020-07-13 14:41:21 +02:00
committed by Andrej Ocenas
parent 5f539230da
commit eedf6e53de
16 changed files with 708 additions and 359 deletions
@@ -0,0 +1,12 @@
import { DataFrame, FieldType } from '../types/dataFrame';
export const isTimeSerie = (frame: DataFrame): boolean => {
if (frame.fields.length > 2) {
return false;
}
return !!frame.fields.find(field => field.type === FieldType.time);
};
export const isTimeSeries = (data: DataFrame[]): boolean => {
return !data.find(frame => !isTimeSerie(frame));
};
@@ -8,10 +8,11 @@ import { filterFramesByRefIdTransformer } from './transformers/filterByRefId';
import { orderFieldsTransformer } from './transformers/order';
import { organizeFieldsTransformer } from './transformers/organize';
import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
import { seriesToRowsTransformer } from './transformers/seriesToRows';
import { renameFieldsTransformer } from './transformers/rename';
import { labelsToFieldsTransformer } from './transformers/labelsToFields';
import { ensureColumnsTransformer } from './transformers/ensureColumns';
import { mergeTransformer } from './transformers/merge/merge';
import { mergeTransformer } from './transformers/merge';
export const standardTransformers = {
noopTransformer,
@@ -25,6 +26,7 @@ export const standardTransformers = {
reduceTransformer,
calculateFieldTransformer,
seriesToColumnsTransformer,
seriesToRowsTransformer,
renameFieldsTransformer,
labelsToFieldsTransformer,
ensureColumnsTransformer,
@@ -8,6 +8,7 @@ export enum DataTransformerID {
rename = 'rename',
calculateField = 'calculateField',
seriesToColumns = 'seriesToColumns',
seriesToRows = 'seriesToRows',
merge = 'merge',
labelsToFields = 'labelsToFields',
filterFields = 'filterFields',
@@ -1,9 +1,9 @@
import { mockTransformationsRegistry } from '../../../utils/tests/mockTransformationsRegistry';
import { DataTransformerConfig, Field, FieldType } from '../../../types';
import { DataTransformerID } from '../ids';
import { toDataFrame } from '../../../dataframe';
import { transformDataFrame } from '../../transformDataFrame';
import { ArrayVector } from '../../../vector';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { DataTransformerConfig, Field, FieldType } from '../../types';
import { DataTransformerID } from './ids';
import { toDataFrame } from '../../dataframe';
import { transformDataFrame } from '../transformDataFrame';
import { ArrayVector } from '../../vector';
import { mergeTransformer, MergeTransformerOptions } from './merge';
describe('Merge multipe to single', () => {
@@ -35,12 +35,11 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 2000]),
createField('Metric', FieldType.string, ['A', 'B']),
createField('Value', FieldType.number, [1, -1]),
createField('Time', FieldType.time, [2000, 1000]),
createField('Temp', FieldType.number, [-1, 1]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine two series with multiple values into one', () => {
@@ -67,12 +66,11 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]),
createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']),
createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]),
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine three series into one', () => {
@@ -107,12 +105,11 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]);
const expected: Field[] = [
createField('Time', FieldType.time, [500, 1000, 2000]),
createField('Metric', FieldType.string, ['C', 'A', 'B']),
createField('Value', FieldType.number, [2, 1, -1]),
createField('Time', FieldType.time, [2000, 1000, 500]),
createField('Temp', FieldType.number, [-1, 1, 2]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine one serie and two tables into one table', () => {
@@ -149,13 +146,12 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [tableA, seriesB, tableB]);
const expected: Field[] = [
createField('Time', FieldType.time, [500, 1000, 1000]),
createField('Metric', FieldType.string, ['C', 'A', 'B']),
createField('Temp', FieldType.number, [2, 1, -1]),
createField('Humidity', FieldType.number, [5, 10, null]),
createField('Time', FieldType.time, [1000, 1000, 500]),
createField('Temp', FieldType.number, [1, -1, 2]),
createField('Humidity', FieldType.number, [10, null, 5]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine one serie and two tables with ISO dates into one table', () => {
@@ -192,13 +188,12 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [tableA, seriesB, tableC]);
const expected: Field[] = [
createField('Time', FieldType.time, ['2019-09-01T11:10:23Z', '2019-10-01T11:10:23Z', '2019-11-01T11:10:23Z']),
createField('Metric', FieldType.string, ['B', 'A', 'C']),
createField('Temp', FieldType.number, [-1, 1, 2]),
createField('Humidity', FieldType.number, [null, 10, 5]),
createField('Time', FieldType.time, ['2019-11-01T11:10:23Z', '2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z']),
createField('Temp', FieldType.number, [2, 1, -1]),
createField('Humidity', FieldType.number, [5, 10, null]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine three tables with multiple values into one', () => {
@@ -235,14 +230,15 @@ describe('Merge multipe to single', () => {
});
const result = transformDataFrame([cfg], [tableA, tableB, tableC]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 100, 100, 124, 125, 126, 149, 150, 200]),
createField('Temp', FieldType.number, [1, -1, 1, 4, 2, 3, 5, 4, 5]),
createField('Humidity', FieldType.number, [10, null, 22, 25, null, null, 30, 14, 55]),
createField('Enabled', FieldType.boolean, [null, true, null, null, false, true, null, null, null]),
createField('Time', FieldType.time, [200, 150, 149, 126, 125, 124, 100, 100, 100]),
createField('Temp', FieldType.number, [5, 4, 5, 3, 2, 4, 1, -1, 1]),
createField('Humidity', FieldType.number, [55, 14, 30, null, null, 25, 10, null, 22]),
createField('Enabled', FieldType.boolean, [null, null, null, true, false, null, null, true, null]),
];
expect(result[0].fields).toMatchObject(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine two time series, where first serie fields has displayName, into one', () => {
@@ -269,13 +265,14 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]),
createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']),
createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]),
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
expect(result[0].fields[2].config).toEqual({});
expect(result[0].fields).toMatchObject(expected);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
});
it('combine two time series, where first serie fields has units, into one', () => {
@@ -302,13 +299,14 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]),
createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']),
createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5], { units: 'celsius' }),
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }),
];
expect(result[0].fields[2].config).toEqual({ units: 'celsius' });
expect(result[0].fields).toMatchObject(expected);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
});
it('combine two time series, where second serie fields has units, into one', () => {
@@ -335,16 +333,28 @@ describe('Merge multipe to single', () => {
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]),
createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']),
createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]),
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Temp', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
expect(result[0].fields[2].config).toEqual({});
expect(result[0].fields).toMatchObject(expected);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
});
});
const createField = (name: string, type: FieldType, values: any[], config = {}): Field => {
return { name, type, values: new ArrayVector(values), config, labels: undefined };
};
const unwrap = (fields: Field[]): Field[] => {
return fields.map(field =>
createField(
field.name,
field.type,
field.values.toArray().map((value: any) => value),
field.config
)
);
};
@@ -0,0 +1,216 @@
import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import { DataFrame, Field, FieldType } from '../../types/dataFrame';
import { omit } from 'lodash';
import { ArrayVector } from '../../vector/ArrayVector';
import { MutableDataFrame, sortDataFrame } from '../../dataframe';
type MergeDetailsKeyFactory = (existing: Record<string, any>, value: Record<string, any>) => string;
export interface MergeTransformerOptions {}
export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
id: DataTransformerID.merge,
name: 'Merge series/tables',
description: 'Merges multiple series/tables into a single serie/table',
defaultOptions: {},
transformer: (options: MergeTransformerOptions) => {
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length <= 1) {
return data;
}
const fieldByName = new Set<string>();
const fieldIndexByName: Record<string, Record<number, number>> = {};
const fieldNamesForKey: string[] = [];
const dataFrame = new MutableDataFrame();
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (!fieldByName.has(field.name)) {
dataFrame.addField(copyFieldStructure(field));
fieldByName.add(field.name);
}
fieldIndexByName[field.name] = fieldIndexByName[field.name] || {};
fieldIndexByName[field.name][frameIndex] = fieldIndex;
if (data.length - 1 !== frameIndex) {
continue;
}
if (Object.keys(fieldIndexByName[field.name]).length === data.length) {
fieldNamesForKey.push(field.name);
}
}
}
if (fieldNamesForKey.length === 0) {
return data;
}
const dataFrameIndexByKey: Record<string, number> = {};
const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey);
const detailsKeyFactory = createDetailsKeyFactory(fieldByName, fieldNamesForKey);
const valueMapper = createValueMapper(data, fieldByName, fieldIndexByName);
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const key = keyFactory(frameIndex, valueIndex);
const value = valueMapper(frameIndex, valueIndex);
mergeOrAdd(key, value, dataFrame, dataFrameIndexByKey, detailsKeyFactory);
}
}
const timeIndex = dataFrame.fields.findIndex(field => field.type === FieldType.time);
if (typeof timeIndex === 'number') {
return [sortDataFrame(dataFrame, timeIndex, true)];
}
return [dataFrame];
};
},
};
const copyFieldStructure = (field: Field): Field => {
return {
...omit(field, ['values', 'state', 'labels', 'config']),
values: new ArrayVector(),
config: {
...omit(field.config, 'displayName'),
},
};
};
const createKeyFactory = (
data: DataFrame[],
fieldPointerByName: Record<string, Record<string, number>>,
keyFieldNames: string[]
) => {
const factoryIndex = keyFieldNames.reduce((index: Record<string, number[]>, fieldName) => {
return Object.keys(fieldPointerByName[fieldName]).reduce((index: Record<string, number[]>, frameIndex) => {
index[frameIndex] = index[frameIndex] || [];
index[frameIndex].push(fieldPointerByName[fieldName][frameIndex]);
return index;
}, index);
}, {});
return (frameIndex: number, valueIndex: number): string => {
return factoryIndex[frameIndex].reduce((key: string, fieldIndex: number) => {
return key + data[frameIndex].fields[fieldIndex].values.get(valueIndex);
}, '');
};
};
const createDetailsKeyFactory = (fieldByName: Set<string>, fieldNamesForKey: string[]): MergeDetailsKeyFactory => {
const fieldNamesToExclude = fieldNamesForKey.reduce((exclude: Record<string, boolean>, fieldName: string) => {
exclude[fieldName] = true;
return exclude;
}, {});
const checkOrder = Array.from(fieldByName).filter(fieldName => !fieldNamesToExclude[fieldName]);
return (existing: Record<string, any>, value: Record<string, any>) => {
return checkOrder.reduce((key: string, fieldName: string) => {
if (typeof existing[fieldName] === 'undefined') {
return key;
}
if (typeof value[fieldName] === 'undefined') {
return key;
}
if (existing[fieldName] === value[fieldName]) {
return key;
}
return key + value[fieldName];
}, '');
};
};
const createValueMapper = (
data: DataFrame[],
fieldByName: Set<string>,
fieldIndexByName: Record<string, Record<number, number>>
) => {
return (frameIndex: number, valueIndex: number) => {
const value: Record<string, any> = {};
const fieldNames = Array.from(fieldByName);
for (const fieldName of fieldNames) {
const fieldIndexByFrameIndex = fieldIndexByName[fieldName];
if (!fieldIndexByFrameIndex) {
continue;
}
const fieldIndex = fieldIndexByFrameIndex[frameIndex];
if (typeof fieldIndex !== 'number') {
continue;
}
const frame = data[frameIndex];
if (!frame || !frame.fields) {
continue;
}
const field = frame.fields[fieldIndex];
if (!field || !field.values) {
continue;
}
value[fieldName] = field.values.get(valueIndex);
}
return value;
};
};
const isMergable = (existing: Record<string, any>, value: Record<string, any>): boolean => {
let mergable = true;
for (const prop in value) {
if (typeof existing[prop] === 'undefined') {
continue;
}
if (existing[prop] === null) {
continue;
}
if (existing[prop] !== value[prop]) {
mergable = false;
break;
}
}
return mergable;
};
const mergeOrAdd = (
key: string,
value: Record<string, any>,
dataFrame: MutableDataFrame,
dataFrameIndexByKey: Record<string, number>,
detailsKeyFactory: MergeDetailsKeyFactory
) => {
if (typeof dataFrameIndexByKey[key] === 'undefined') {
dataFrame.add(value);
dataFrameIndexByKey[key] = dataFrame.length - 1;
return;
}
const dataFrameIndex = dataFrameIndexByKey[key];
const existing = dataFrame.get(dataFrameIndex);
if (isMergable(existing, value)) {
const merged = { ...existing, ...value };
dataFrame.set(dataFrameIndex, merged);
return;
}
const nextKey = key + detailsKeyFactory(existing, value);
mergeOrAdd(nextKey, value, dataFrame, dataFrameIndexByKey, detailsKeyFactory);
};
@@ -1,135 +0,0 @@
import { MutableDataFrame } from '../../../dataframe';
import {
DataFrame,
FieldType,
Field,
TIME_SERIES_TIME_FIELD_NAME,
TIME_SERIES_VALUE_FIELD_NAME,
} from '../../../types/dataFrame';
import { ArrayVector } from '../../../vector';
import { omit } from 'lodash';
import { getFrameDisplayName } from '../../../field';
interface DataFrameBuilderResult {
dataFrame: MutableDataFrame;
valueMapper: ValueMapper;
}
type ValueMapper = (frame: DataFrame, valueIndex: number, timeIndex: number) => Record<string, any>;
const TIME_SERIES_METRIC_FIELD_NAME = 'Metric';
export class DataFrameBuilder {
private isOnlyTimeSeries: boolean;
private displayMetricField: boolean;
private valueFields: Record<string, Field>;
private timeField: Field | null;
constructor() {
this.isOnlyTimeSeries = true;
this.displayMetricField = false;
this.valueFields = {};
this.timeField = null;
}
addFields(frame: DataFrame, timeIndex: number): void {
if (frame.fields.length > 2) {
this.isOnlyTimeSeries = false;
}
if (frame.fields.length === 2) {
this.displayMetricField = true;
}
for (let index = 0; index < frame.fields.length; index++) {
const field = frame.fields[index];
if (index === timeIndex) {
if (!this.timeField) {
this.timeField = this.copyStructure(field, TIME_SERIES_TIME_FIELD_NAME);
}
continue;
}
if (!this.valueFields[field.name]) {
this.valueFields[field.name] = this.copyStructure(field, field.name);
}
}
}
build(): DataFrameBuilderResult {
return {
dataFrame: this.createDataFrame(),
valueMapper: this.createValueMapper(),
};
}
private createValueMapper(): ValueMapper {
return (frame: DataFrame, valueIndex: number, timeIndex: number) => {
return frame.fields.reduce((values: Record<string, any>, field, index) => {
const value = field.values.get(valueIndex);
if (index === timeIndex) {
values[TIME_SERIES_TIME_FIELD_NAME] = value;
if (this.displayMetricField) {
values[TIME_SERIES_METRIC_FIELD_NAME] = getFrameDisplayName(frame);
}
return values;
}
if (this.isOnlyTimeSeries) {
values[TIME_SERIES_VALUE_FIELD_NAME] = value;
return values;
}
values[field.name] = value;
return values;
}, {});
};
}
private createDataFrame(): MutableDataFrame {
const dataFrame = new MutableDataFrame();
if (this.timeField) {
dataFrame.addField(this.timeField);
if (this.displayMetricField) {
dataFrame.addField({
name: TIME_SERIES_METRIC_FIELD_NAME,
type: FieldType.string,
});
}
}
const valueFields = Object.values(this.valueFields);
if (this.isOnlyTimeSeries) {
if (valueFields.length > 0) {
dataFrame.addField({
...valueFields[0],
name: TIME_SERIES_VALUE_FIELD_NAME,
});
}
return dataFrame;
}
for (const field of valueFields) {
dataFrame.addField(field);
}
return dataFrame;
}
private copyStructure(field: Field, name: string): Field {
return {
...omit(field, ['values', 'name', 'state', 'labels', 'config']),
name,
values: new ArrayVector(),
config: {
...omit(field.config, 'displayName'),
},
};
}
}
@@ -1,74 +0,0 @@
import { DataFrame } from '../../../types/dataFrame';
import { timeComparer } from '../../../field/fieldComparers';
import { sortDataFrame } from '../../../dataframe';
import { TimeFieldsByFrame } from './TimeFieldsByFrame';
interface DataFrameStackValue {
valueIndex: number;
timeIndex: number;
frame: DataFrame;
}
export class DataFramesStackedByTime {
private valuesPointerByFrame: Record<number, number>;
private dataFrames: DataFrame[];
private isSorted: boolean;
constructor(private timeFields: TimeFieldsByFrame) {
this.valuesPointerByFrame = {};
this.dataFrames = [];
this.isSorted = false;
}
push(frame: DataFrame): number {
const index = this.dataFrames.length;
this.valuesPointerByFrame[index] = 0;
this.dataFrames.push(frame);
return index;
}
pop(): DataFrameStackValue {
if (!this.isSorted) {
this.sortByTime();
this.isSorted = true;
}
const frameIndex = this.dataFrames.reduce((champion, frame, index) => {
const championTime = this.peekTimeValueForFrame(champion);
const contenderTime = this.peekTimeValueForFrame(index);
return timeComparer(contenderTime, championTime) >= 0 ? champion : index;
}, 0);
const previousPointer = this.movePointerForward(frameIndex);
return {
frame: this.dataFrames[frameIndex],
valueIndex: previousPointer,
timeIndex: this.timeFields.getFieldIndex(frameIndex),
};
}
getLength(): number {
const frames = Object.values(this.dataFrames);
return frames.reduce((length: number, frame) => (length += frame.length), 0);
}
private peekTimeValueForFrame(frameIndex: number): any {
const timeField = this.timeFields.getField(frameIndex);
const valuePointer = this.valuesPointerByFrame[frameIndex];
return timeField.values.get(valuePointer);
}
private movePointerForward(frameIndex: number): number {
const currentPointer = this.valuesPointerByFrame[frameIndex];
this.valuesPointerByFrame[frameIndex] = currentPointer + 1;
return currentPointer;
}
private sortByTime() {
this.dataFrames = this.dataFrames.map((frame, index) => {
const timeFieldIndex = this.timeFields.getFieldIndex(index);
return sortDataFrame(frame, timeFieldIndex);
});
}
}
@@ -1,39 +0,0 @@
import { isNumber } from 'lodash';
import { Field, DataFrame } from '../../../types/dataFrame';
import { getTimeField } from '../../../dataframe';
export class TimeFieldsByFrame {
private timeIndexByFrameIndex: Record<number, number>;
private timeFieldByFrameIndex: Record<number, Field>;
constructor() {
this.timeIndexByFrameIndex = {};
this.timeFieldByFrameIndex = {};
}
add(frameIndex: number, frame: DataFrame): void {
const fieldDescription = getTimeField(frame);
const timeIndex = fieldDescription?.timeIndex;
const timeField = fieldDescription?.timeField;
if (isNumber(timeIndex)) {
this.timeIndexByFrameIndex[frameIndex] = timeIndex;
}
if (timeField) {
this.timeFieldByFrameIndex[frameIndex] = timeField;
}
}
getField(frameIndex: number): Field {
return this.timeFieldByFrameIndex[frameIndex];
}
getFieldIndex(frameIndex: number): number {
return this.timeIndexByFrameIndex[frameIndex];
}
getLength() {
return Object.keys(this.timeIndexByFrameIndex).length;
}
}
@@ -1,47 +0,0 @@
import { DataTransformerID } from '../ids';
import { DataTransformerInfo } from '../../../types/transformations';
import { DataFrame } from '../../../types/dataFrame';
import { DataFrameBuilder } from './DataFrameBuilder';
import { TimeFieldsByFrame } from './TimeFieldsByFrame';
import { DataFramesStackedByTime } from './DataFramesStackedByTime';
export interface MergeTransformerOptions {}
export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
id: DataTransformerID.merge,
name: 'Merge series/tables',
description: 'Merges multiple series/tables by time into a single serie/table',
defaultOptions: {},
transformer: (options: MergeTransformerOptions) => {
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length <= 1) {
return data;
}
const timeFields = new TimeFieldsByFrame();
const framesStack = new DataFramesStackedByTime(timeFields);
const dataFrameBuilder = new DataFrameBuilder();
for (const frame of data) {
const frameIndex = framesStack.push(frame);
timeFields.add(frameIndex, frame);
const timeIndex = timeFields.getFieldIndex(frameIndex);
dataFrameBuilder.addFields(frame, timeIndex);
}
if (data.length !== timeFields.getLength()) {
return data;
}
const { dataFrame, valueMapper } = dataFrameBuilder.build();
for (let index = 0; index < framesStack.getLength(); index++) {
const { frame, valueIndex, timeIndex } = framesStack.pop();
dataFrame.add(valueMapper(frame, valueIndex, timeIndex));
}
return [dataFrame];
};
},
};
@@ -0,0 +1,237 @@
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { DataTransformerConfig, Field, FieldType } from '../../types';
import { DataTransformerID } from './ids';
import { toDataFrame } from '../../dataframe';
import { transformDataFrame } from '../transformDataFrame';
import { ArrayVector } from '../../vector';
import { seriesToRowsTransformer, SeriesToRowsTransformerOptions } from './seriesToRows';
describe('Series to rows', () => {
beforeAll(() => {
mockTransformationsRegistry([seriesToRowsTransformer]);
});
it('combine two series into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const seriesA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [1000] },
{ name: 'Temp', type: FieldType.number, values: [1] },
],
});
const seriesB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [2000] },
{ name: 'Temp', type: FieldType.number, values: [-1] },
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000]),
createField('Metric', FieldType.string, ['B', 'A']),
createField('Value', FieldType.number, [-1, 1]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine two series with multiple values into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const seriesA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 150, 200] },
{ name: 'Temp', type: FieldType.number, values: [1, 4, 5] },
],
});
const seriesB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 125, 126] },
{ name: 'Temp', type: FieldType.number, values: [-1, 2, 3] },
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine three series into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const seriesA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [1000] },
{ name: 'Temp', type: FieldType.number, values: [1] },
],
});
const seriesB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [2000] },
{ name: 'Temp', type: FieldType.number, values: [-1] },
],
});
const seriesC = toDataFrame({
name: 'C',
fields: [
{ name: 'Time', type: FieldType.time, values: [500] },
{ name: 'Temp', type: FieldType.number, values: [2] },
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]);
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000, 500]),
createField('Metric', FieldType.string, ['B', 'A', 'C']),
createField('Value', FieldType.number, [-1, 1, 2]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
});
it('combine two time series, where first serie fields has displayName, into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const serieA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 150, 200], config: { displayName: 'Random time' } },
{ name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { displayName: 'Temp' } },
],
});
const serieB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 125, 126] },
{ name: 'Temp', type: FieldType.number, values: [-1, 2, 3] },
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
});
it('combine two time series, where first serie fields has units, into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const serieA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 150, 200] },
{ name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { units: 'celsius' } },
],
});
const serieB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 125, 126] },
{ name: 'Temp', type: FieldType.number, values: [-1, 2, 3] },
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }),
];
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
});
it('combine two time series, where second serie fields has units, into one', () => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
};
const serieA = toDataFrame({
name: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 150, 200] },
{ name: 'Temp', type: FieldType.number, values: [1, 4, 5] },
],
});
const serieB = toDataFrame({
name: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 125, 126] },
{ name: 'Temp', type: FieldType.number, values: [-1, 2, 3], config: { units: 'celsius' } },
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
});
});
const createField = (name: string, type: FieldType, values: any[], config = {}): Field => {
return { name, type, values: new ArrayVector(values), config, labels: undefined };
};
const unwrap = (fields: Field[]): Field[] => {
return fields.map(field =>
createField(
field.name,
field.type,
field.values.toArray().map((value: any) => value),
field.config
)
);
};
@@ -0,0 +1,97 @@
import { omit } from 'lodash';
import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import {
DataFrame,
Field,
FieldType,
TIME_SERIES_TIME_FIELD_NAME,
TIME_SERIES_VALUE_FIELD_NAME,
TIME_SERIES_METRIC_FIELD_NAME,
} from '../../types/dataFrame';
import { isTimeSeries } from '../../dataframe/utils';
import { MutableDataFrame, sortDataFrame } from '../../dataframe';
import { ArrayVector } from '../../vector';
import { getFrameDisplayName } from '../../field/fieldState';
export interface SeriesToRowsTransformerOptions {}
export const seriesToRowsTransformer: DataTransformerInfo<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
name: 'Series to rows',
description: 'Combines multiple series into a single serie and appends a column with metric name per value.',
defaultOptions: {},
transformer: (options: SeriesToRowsTransformerOptions) => {
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length <= 1) {
return data;
}
if (!isTimeSeries(data)) {
return data;
}
const timeFieldByIndex: Record<number, number> = {};
const targetFields = new Set<string>();
const dataFrame = new MutableDataFrame();
const metricField: Field = {
name: TIME_SERIES_METRIC_FIELD_NAME,
values: new ArrayVector(),
config: {},
type: FieldType.string,
};
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (field.type === FieldType.time) {
timeFieldByIndex[frameIndex] = fieldIndex;
if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME));
dataFrame.addField(metricField);
targetFields.add(TIME_SERIES_TIME_FIELD_NAME);
}
continue;
}
if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME));
targetFields.add(TIME_SERIES_VALUE_FIELD_NAME);
}
}
}
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const timeFieldIndex = timeFieldByIndex[frameIndex];
const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0;
dataFrame.add({
[TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex),
[TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame),
[TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex),
});
}
}
return [sortDataFrame(dataFrame, 0, true)];
};
},
};
const copyFieldStructure = (field: Field, name: string): Field => {
return {
...omit(field, ['values', 'state', 'labels', 'config', 'name']),
name: name,
values: new ArrayVector(),
config: {
...omit(field.config, 'displayName'),
},
};
};
@@ -150,3 +150,4 @@ export interface FieldCalcs extends Record<string, any> {}
export const TIME_SERIES_VALUE_FIELD_NAME = 'Value';
export const TIME_SERIES_TIME_FIELD_NAME = 'Time';
export const TIME_SERIES_METRIC_FIELD_NAME = 'Metric';