From fe423644aef7b307b8222019023deae555d83143 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 14 Oct 2020 10:34:04 -0700 Subject: [PATCH] Transformations: add Concatenate fields transformer (#28237) --- .../src/transformations/transformers.ts | 2 + .../transformers/concat.test.ts | 136 ++++++++++++++++++ .../transformations/transformers/concat.ts | 105 ++++++++++++++ .../src/transformations/transformers/ids.ts | 1 + packages/grafana-data/src/types/datasource.ts | 2 +- packages/grafana-data/src/types/live.ts | 18 +-- packages/grafana-runtime/src/services/live.ts | 4 +- .../grafana-ui/src/components/Monaco/types.ts | 2 +- .../ConcatenateTransformerEditor.tsx | 93 ++++++++++++ public/app/core/utils/standardTransformers.ts | 2 + 10 files changed, 352 insertions(+), 13 deletions(-) create mode 100644 packages/grafana-data/src/transformations/transformers/concat.test.ts create mode 100644 packages/grafana-data/src/transformations/transformers/concat.ts create mode 100644 public/app/core/components/TransformersUI/ConcatenateTransformerEditor.tsx diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index 51976451f38..e2e95cbdbc9 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -1,5 +1,6 @@ import { appendTransformer } from './transformers/append'; import { reduceTransformer } from './transformers/reduce'; +import { concatenateTransformer } from './transformers/concat'; import { calculateFieldTransformer } from './transformers/calculateField'; import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter'; import { filterFieldsByNameTransformer } from './transformers/filterByName'; @@ -25,6 +26,7 @@ export const standardTransformers = { organizeFieldsTransformer, appendTransformer, reduceTransformer, + concatenateTransformer, calculateFieldTransformer, seriesToColumnsTransformer, seriesToRowsTransformer, diff --git a/packages/grafana-data/src/transformations/transformers/concat.test.ts b/packages/grafana-data/src/transformations/transformers/concat.test.ts new file mode 100644 index 00000000000..8e90b07ee46 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/concat.test.ts @@ -0,0 +1,136 @@ +import { toDataFrame } from '../../dataframe/processDataFrame'; +import { concatenateFields, ConcatenateFrameNameMode } from './concat'; + +export const simpleABC = toDataFrame({ + name: 'ABC', + fields: [ + { name: 'A', values: [1, 2] }, + { name: 'B', values: [1, 2] }, + { name: 'C', values: [1, 2] }, + ], +}); + +export const simpleXYZ = toDataFrame({ + name: 'XYZ', + fields: [ + { name: 'X', values: [1, 2, 3] }, + { name: 'Y', values: [1, 2, 3] }, + { name: 'Z', values: [1, 2, 3] }, + ], +}); + +describe('Concat Transformer', () => { + it('dropping frame name', () => { + const frame = concatenateFields([simpleABC, simpleXYZ], { frameNameMode: ConcatenateFrameNameMode.Drop }); + expect(frame.length).toBe(3); + expect(frame.fields.map(f => ({ name: f.name, labels: f.labels }))).toMatchInlineSnapshot(` + Array [ + Object { + "labels": undefined, + "name": "A", + }, + Object { + "labels": undefined, + "name": "B", + }, + Object { + "labels": undefined, + "name": "C", + }, + Object { + "labels": undefined, + "name": "X", + }, + Object { + "labels": undefined, + "name": "Y", + }, + Object { + "labels": undefined, + "name": "Z", + }, + ] + `); + }); + + it('using field name', () => { + const frame = concatenateFields([simpleABC, simpleXYZ], { frameNameMode: ConcatenateFrameNameMode.FieldName }); + expect(frame.length).toBe(3); + expect(frame.fields.map(f => ({ name: f.name, labels: f.labels }))).toMatchInlineSnapshot(` + Array [ + Object { + "labels": undefined, + "name": "ABC · A", + }, + Object { + "labels": undefined, + "name": "ABC · B", + }, + Object { + "labels": undefined, + "name": "ABC · C", + }, + Object { + "labels": undefined, + "name": "XYZ · X", + }, + Object { + "labels": undefined, + "name": "XYZ · Y", + }, + Object { + "labels": undefined, + "name": "XYZ · Z", + }, + ] + `); + }); + + it('using field label', () => { + const frame = concatenateFields([simpleABC, simpleXYZ], { + frameNameMode: ConcatenateFrameNameMode.Label, + frameNameLabel: 'sensor', + }); + expect(frame.length).toBe(3); + expect(frame.fields.map(f => ({ name: f.name, labels: f.labels }))).toMatchInlineSnapshot(` + Array [ + Object { + "labels": Object { + "sensor": "ABC", + }, + "name": "A", + }, + Object { + "labels": Object { + "sensor": "ABC", + }, + "name": "B", + }, + Object { + "labels": Object { + "sensor": "ABC", + }, + "name": "C", + }, + Object { + "labels": Object { + "sensor": "XYZ", + }, + "name": "X", + }, + Object { + "labels": Object { + "sensor": "XYZ", + }, + "name": "Y", + }, + Object { + "labels": Object { + "sensor": "XYZ", + }, + "name": "Z", + }, + ] + `); + }); +}); diff --git a/packages/grafana-data/src/transformations/transformers/concat.ts b/packages/grafana-data/src/transformations/transformers/concat.ts new file mode 100644 index 00000000000..f6ede040060 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/concat.ts @@ -0,0 +1,105 @@ +import { map } from 'rxjs/operators'; + +import { DataTransformerID } from './ids'; +import { DataTransformerInfo } from '../../types/transformations'; +import { DataFrame, Field } from '../../types/dataFrame'; +import { ArrayVector } from '../../vector'; + +export enum ConcatenateFrameNameMode { + /** + * Ignore the source frame name when moving to the destination + */ + Drop = 'drop', + + /** + * Copy the source frame name to the destination field. The final field will contain + * both the frame and field name + */ + FieldName = 'field', + + /** + * Copy the source frame name to a label on the field. The label key is controlled + * by frameNameLabel + */ + Label = 'label', +} + +export interface ConcatenateTransformerOptions { + frameNameMode?: ConcatenateFrameNameMode; + frameNameLabel?: string; +} + +export const concatenateTransformer: DataTransformerInfo = { + id: DataTransformerID.concatenate, + name: 'Concatenate fields', + description: + 'Combine all fields into a single frame. Values will be appended with undefined values if not the same length.', + defaultOptions: { + frameNameMode: ConcatenateFrameNameMode.FieldName, + frameNameLabel: 'frame', + }, + operator: options => source => + source.pipe( + map(dataFrames => { + if (!Array.isArray(dataFrames) || dataFrames.length < 2) { + return dataFrames; // noop with single frame + } + return [concatenateFields(dataFrames, options)]; + }) + ), +}; + +/** + * @internal only exported for tests + */ +export function concatenateFields(data: DataFrame[], opts: ConcatenateTransformerOptions): DataFrame { + let sameLength = true; + let maxLength = data[0].length; + const frameNameLabel = opts.frameNameLabel ?? 'frame'; + let fields: Field[] = []; + + for (const frame of data) { + if (maxLength !== frame.length) { + sameLength = false; + maxLength = Math.max(maxLength, frame.length); + } + + for (const f of frame.fields) { + const copy = { ...f }; + copy.state = undefined; + if (frame.name) { + if (opts.frameNameMode === ConcatenateFrameNameMode.Drop) { + // nothing -- skip the name + } else if (opts.frameNameMode === ConcatenateFrameNameMode.Label) { + copy.labels = { ...f.labels }; + copy.labels[frameNameLabel] = frame.name; + } else if (!copy.name || copy.name === 'Value') { + copy.name = frame.name; + } else { + copy.name = `${frame.name} · ${f.name}`; + } + } + fields.push(copy); + } + } + + // Make sure all fields have the same length + if (!sameLength) { + fields = fields.map(f => { + if (f.values.length === maxLength) { + return f; + } + const values = f.values.toArray(); + values.length = maxLength; + return { + ...f, + values: new ArrayVector(values), + }; + }); + } + + return { + fields, + length: maxLength, + }; +} diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index b03975c749a..78523501e4a 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -10,6 +10,7 @@ export enum DataTransformerID { seriesToColumns = 'seriesToColumns', seriesToRows = 'seriesToRows', merge = 'merge', + concatenate = 'concatenate', labelsToFields = 'labelsToFields', filterFields = 'filterFields', filterFieldsByName = 'filterFieldsByName', diff --git a/packages/grafana-data/src/types/datasource.ts b/packages/grafana-data/src/types/datasource.ts index 4b7f6d529d6..8abd53ca26e 100644 --- a/packages/grafana-data/src/types/datasource.ts +++ b/packages/grafana-data/src/types/datasource.ts @@ -291,7 +291,7 @@ export abstract class DataSourceApi< * * Note: `plugin.json` must also define `live: true` * - * @experimental + * @alpha -- experimental */ channelSupport?: LiveChannelSupport; } diff --git a/packages/grafana-data/src/types/live.ts b/packages/grafana-data/src/types/live.ts index c806993f0dc..e64d92c2878 100644 --- a/packages/grafana-data/src/types/live.ts +++ b/packages/grafana-data/src/types/live.ts @@ -15,7 +15,7 @@ export enum LiveChannelScope { } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannelConfig { /** @@ -69,7 +69,7 @@ export enum LiveChannelEventType { } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannelStatusEvent { type: LiveChannelEventType.Status; @@ -101,12 +101,12 @@ export interface LiveChannelStatusEvent { export interface LiveChannelJoinEvent { type: LiveChannelEventType.Join; - user: any; // @experimental -- will be filled in when we improve the UI + user: any; // @alpha -- experimental -- will be filled in when we improve the UI } export interface LiveChannelLeaveEvent { type: LiveChannelEventType.Leave; - user: any; // @experimental -- will be filled in when we improve the UI + user: any; // @alpha -- experimental -- will be filled in when we improve the UI } export interface LiveChannelMessageEvent { @@ -137,14 +137,14 @@ export function isLiveChannelMessageEvent(evt: LiveChannelEvent): evt is L } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannelPresenceStatus { - users: any; // @experimental -- will be filled in when we improve the UI + users: any; // @alpha -- experimental -- will be filled in when we improve the UI } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannelAddress { scope: LiveChannelScope; @@ -160,7 +160,7 @@ export function isValidLiveChannelAddress(addr?: LiveChannelAddress): addr is Li } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannel { /** The fully qualified channel id: ${scope}/${namespace}/${path} */ @@ -201,7 +201,7 @@ export interface LiveChannel { } /** - * @experimental + * @alpha -- experimental */ export interface LiveChannelSupport { /** diff --git a/packages/grafana-runtime/src/services/live.ts b/packages/grafana-runtime/src/services/live.ts index df39df6f1bd..c57a4a1777f 100644 --- a/packages/grafana-runtime/src/services/live.ts +++ b/packages/grafana-runtime/src/services/live.ts @@ -2,7 +2,7 @@ import { LiveChannel, LiveChannelAddress } from '@grafana/data'; import { Observable } from 'rxjs'; /** - * @experimental + * @alpha -- experimental */ export interface GrafanaLiveSrv { /** @@ -42,7 +42,7 @@ export const setGrafanaLiveSrv = (instance: GrafanaLiveSrv) => { * Used to retrieve the {@link GrafanaLiveSrv} that allows you to subscribe to * server side events and streams * - * @experimental + * @alpha -- experimental * @public */ export const getGrafanaLiveSrv = (): GrafanaLiveSrv => singletonInstance; diff --git a/packages/grafana-ui/src/components/Monaco/types.ts b/packages/grafana-ui/src/components/Monaco/types.ts index d2dd6ea8978..33bde5e7952 100644 --- a/packages/grafana-ui/src/components/Monaco/types.ts +++ b/packages/grafana-ui/src/components/Monaco/types.ts @@ -14,7 +14,7 @@ export interface CodeEditorProps { /** * Callback after the editor has mounted that gives you raw access to monaco * - * @experimental - real type is: monaco.editor.IStandaloneCodeEditor + * @alpha -- experimental - real type is: monaco.editor.IStandaloneCodeEditor */ onEditorDidMount?: (editor: any) => void; diff --git a/public/app/core/components/TransformersUI/ConcatenateTransformerEditor.tsx b/public/app/core/components/TransformersUI/ConcatenateTransformerEditor.tsx new file mode 100644 index 00000000000..fc23664c78d --- /dev/null +++ b/public/app/core/components/TransformersUI/ConcatenateTransformerEditor.tsx @@ -0,0 +1,93 @@ +import React, { ChangeEvent } from 'react'; +import { + DataTransformerID, + SelectableValue, + standardTransformers, + TransformerRegistyItem, + TransformerUIProps, +} from '@grafana/data'; +import { Input, Select } from '@grafana/ui'; +import { + ConcatenateTransformerOptions, + ConcatenateFrameNameMode, +} from '@grafana/data/src/transformations/transformers/concat'; + +interface ConcatenateTransformerEditorProps extends TransformerUIProps {} + +const nameModes: Array> = [ + { value: ConcatenateFrameNameMode.FieldName, label: 'Copy frame name to field name' }, + { value: ConcatenateFrameNameMode.Label, label: 'Add a label with the frame name' }, + { value: ConcatenateFrameNameMode.Drop, label: 'Ignore the frame name' }, +]; + +export class ConcatenateTransformerEditor extends React.PureComponent { + constructor(props: ConcatenateTransformerEditorProps) { + super(props); + } + + onModeChanged = (value: SelectableValue) => { + const { options, onChange } = this.props; + const frameNameMode = value.value ?? ConcatenateFrameNameMode.FieldName; + onChange({ + ...options, + frameNameMode, + }); + }; + + onLabelChanged = (evt: ChangeEvent) => { + const { options } = this.props; + this.props.onChange({ + ...options, + frameNameLabel: evt.target.value, + }); + }; + + //--------------------------------------------------------- + // Render + //--------------------------------------------------------- + + render() { + const { options } = this.props; + + const frameNameMode = options.frameNameMode ?? ConcatenateFrameNameMode.FieldName; + + return ( +
+
+
+
Name
+ +
+
+ )} +
+ ); + } +} + +export const concatenateTransformRegistryItem: TransformerRegistyItem = { + id: DataTransformerID.concatenate, + editor: ConcatenateTransformerEditor, + transformation: standardTransformers.concatenateTransformer, + name: 'Concatenate fields', + description: + 'Combine all fields into a single frame. Values will be appended with undefined values if not the same length.', +}; diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts index 62dce00aec8..63773a37ab6 100644 --- a/public/app/core/utils/standardTransformers.ts +++ b/public/app/core/utils/standardTransformers.ts @@ -9,6 +9,7 @@ import { labelsToFieldsTransformerRegistryItem } from '../components/Transformer import { groupByTransformRegistryItem } from '../components/TransformersUI/GroupByTransformerEditor'; import { mergeTransformerRegistryItem } from '../components/TransformersUI/MergeTransformerEditor'; import { seriesToRowsTransformerRegistryItem } from '../components/TransformersUI/SeriesToRowsTransformerEditor'; +import { concatenateTransformRegistryItem } from '../components/TransformersUI/ConcatenateTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -18,6 +19,7 @@ export const getStandardTransformers = (): Array> => organizeFieldsTransformRegistryItem, seriesToFieldsTransformerRegistryItem, seriesToRowsTransformerRegistryItem, + concatenateTransformRegistryItem, calculateFieldTransformRegistryItem, labelsToFieldsTransformerRegistryItem, groupByTransformRegistryItem,