From 883d61d1914a3ee3e533da055f2bbac028e1760e Mon Sep 17 00:00:00 2001 From: Leon Sorokin Date: Thu, 20 Oct 2022 17:22:02 -0500 Subject: [PATCH] Transformers: PartitionByValues (#56767) Co-authored-by: Ryan McKinley --- .../transformers/filterByName.ts | 3 +- .../src/transformations/transformers/ids.ts | 1 + .../src/transformations/transformers/noop.ts | 18 +- .../PartitionByValuesEditor.tsx | 107 ++++++++++ .../partitionByValues/partition.ts | 52 +++++ .../partitionByValues.test.ts | 183 ++++++++++++++++++ .../partitionByValues/partitionByValues.ts | 112 +++++++++++ .../transformers/standardTransformers.ts | 2 + 8 files changed, 467 insertions(+), 11 deletions(-) create mode 100644 public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx create mode 100644 public/app/features/transformers/partitionByValues/partition.ts create mode 100644 public/app/features/transformers/partitionByValues/partitionByValues.test.ts create mode 100644 public/app/features/transformers/partitionByValues/partitionByValues.ts diff --git a/packages/grafana-data/src/transformations/transformers/filterByName.ts b/packages/grafana-data/src/transformations/transformers/filterByName.ts index 7a9ee71c737..f241badb492 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByName.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByName.ts @@ -32,7 +32,8 @@ export const filterFieldsByNameTransformer: DataTransformerInfo { +// Exported to share with other implementations, but not exported to `@grafana/data` +export const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => { if (!options) { return undefined; } diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index 881add081a8..968958d9668 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -35,4 +35,5 @@ export enum DataTransformerID { extractFields = 'extractFields', groupingToMatrix = 'groupingToMatrix', limit = 'limit', + partitionByValues = 'partitionByValues', } diff --git a/packages/grafana-data/src/transformations/transformers/noop.ts b/packages/grafana-data/src/transformations/transformers/noop.ts index ba70bd27b36..bf2d5f93c9b 100644 --- a/packages/grafana-data/src/transformations/transformers/noop.ts +++ b/packages/grafana-data/src/transformations/transformers/noop.ts @@ -1,21 +1,19 @@ -import { DataTransformerInfo } from '../../types/transformations'; +import { DataFrame } from '../../types'; +import { SynchronousDataTransformerInfo } from '../../types/transformations'; import { DataTransformerID } from './ids'; -export interface NoopTransformerOptions { - include?: string; - exclude?: string; -} +export interface NoopTransformerOptions {} -export const noopTransformer: DataTransformerInfo = { +export const noopTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.noop, name: 'noop', description: 'No-operation transformer', defaultOptions: {}, - /** - * Return a modified copy of the series. If the transform is not or should not - * be applied, just return the input series - */ + /** no operation */ operator: (options: NoopTransformerOptions) => (source) => source, + + /** no operation */ + transformer: (options: NoopTransformerOptions) => (data: DataFrame[]) => data, }; diff --git a/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx b/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx new file mode 100644 index 00000000000..96dbda4b5d0 --- /dev/null +++ b/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx @@ -0,0 +1,107 @@ +import React, { useCallback, useMemo } from 'react'; + +import { + DataTransformerID, + PluginState, + TransformerRegistryItem, + TransformerUIProps, + SelectableValue, +} from '@grafana/data'; +import { InlineField, InlineFieldRow, ValuePicker, Button, HorizontalGroup, FieldValidationMessage } from '@grafana/ui'; +import { useFieldDisplayNames, useSelectOptions } from '@grafana/ui/src/components/MatchersUI/utils'; + +import { partitionByValuesTransformer, PartitionByValuesTransformerOptions } from './partitionByValues'; + +export function PartitionByValuesEditor({ + input, + options, + onChange, +}: TransformerUIProps) { + const names = useFieldDisplayNames(input); + const allSelectOptions = useSelectOptions(names); + const selectOptions = useMemo(() => { + const fieldNames = new Set(options.fields); + + if (fieldNames.size < 1) { + return allSelectOptions; + } + return allSelectOptions.filter((v) => !fieldNames.has(v.value!)); + }, [allSelectOptions, options]); + + const addField = useCallback( + (v: SelectableValue) => { + if (!v.value) { + return; + } + + const fieldNames = new Set(options.fields); + + fieldNames.add(v.value); + + onChange({ + ...options, + fields: [...fieldNames], + }); + }, + [onChange, options] + ); + + const removeField = useCallback( + (v: string) => { + if (!v) { + return; + } + + const fieldNames = new Set(options.fields); + + fieldNames.delete(v); + + onChange({ + ...options, + fields: [...fieldNames], + }); + }, + [onChange, options] + ); + + if (input.length > 1) { + return Partition by values only works with a single frame.; + } + + const fieldNames = [...new Set(options.fields)]; + + return ( +
+ + + + {fieldNames.map((name) => ( + + ))} + {selectOptions.length && ( + + )} + + + +
+ ); +} + +export const partitionByValuesTransformRegistryItem: TransformerRegistryItem = { + id: DataTransformerID.partitionByValues, + editor: PartitionByValuesEditor, + transformation: partitionByValuesTransformer, + name: partitionByValuesTransformer.name, + description: partitionByValuesTransformer.description, + state: PluginState.alpha, +}; diff --git a/public/app/features/transformers/partitionByValues/partition.ts b/public/app/features/transformers/partitionByValues/partition.ts new file mode 100644 index 00000000000..3980e5a9e30 --- /dev/null +++ b/public/app/features/transformers/partitionByValues/partition.ts @@ -0,0 +1,52 @@ +type Idxs = number[]; +type KeyMap = Map; +type Accum = Idxs[]; + +/** The totally type-aware flavor is much slower, so we prefer to disable the lint rule in this case */ +/* eslint-disable @typescript-eslint/consistent-type-assertions */ +const digArrs = (map: KeyMap | Idxs, depth: number, acc: Accum = []) => { + // the leaf nodes are always Idxs + if (depth === 0) { + acc.push(map as Idxs); + } + // the branch nodes are always KeyMaps + else { + (map as KeyMap).forEach((v) => { + digArrs(v, depth - 1, acc); + }); + } + + return acc; +}; + +// in: [['a','b','z','b'], ['c','c','x','c']] +// out: [[0], [1,3], [2]] +export function partition(keys: unknown[][]) { + const len = keys[0].length; + const klen = keys.length; + + const rootMap: KeyMap = new Map(); + + for (let i = 0; i < len; i++) { + let cur: KeyMap | Idxs = rootMap; + + for (let j = 0; j < klen; j++) { + let key = keys[j][i]; + + let next: KeyMap | Idxs | undefined = (cur as KeyMap).get(key); + + if (next == null) { + next = j === klen - 1 ? [] : new Map(); + (cur as KeyMap).set(key, next); + } + + cur = next; + } + + (cur as Idxs).push(i); + } + + return digArrs(rootMap, klen); +} + +/* eslint-enable @typescript-eslint/consistent-type-assertions */ diff --git a/public/app/features/transformers/partitionByValues/partitionByValues.test.ts b/public/app/features/transformers/partitionByValues/partitionByValues.test.ts new file mode 100644 index 00000000000..bf229166dc3 --- /dev/null +++ b/public/app/features/transformers/partitionByValues/partitionByValues.test.ts @@ -0,0 +1,183 @@ +import { toDataFrame, FieldType } from '@grafana/data'; + +import { partitionByValuesTransformer, PartitionByValuesTransformerOptions } from './partitionByValues'; + +describe('Partition by values transformer', () => { + it('should partition by one field', () => { + const source = [ + toDataFrame({ + name: 'XYZ', + refId: 'A', + fields: [ + { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] }, + { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] }, + ], + }), + ]; + + const config: PartitionByValuesTransformerOptions = { + fields: ['region'], + }; + + let partitioned = partitionByValuesTransformer.transformer(config)(source); + + expect(partitioned.length).toEqual(2); + + expect(partitioned[0].length).toEqual(3); + expect(partitioned[0].name).toEqual('Europe'); + expect(partitioned[0].fields[0].name).toEqual('model'); + expect(partitioned[0].fields[1].name).toEqual('region'); + expect(partitioned[0].fields[0].values.toArray()).toEqual(['E1', 'E2', 'E3']); + expect(partitioned[0].fields[1].values.toArray()).toEqual(['Europe', 'Europe', 'Europe']); + + expect(partitioned[1].length).toEqual(3); + expect(partitioned[1].name).toEqual('China'); + expect(partitioned[1].fields[0].name).toEqual('model'); + expect(partitioned[1].fields[1].name).toEqual('region'); + expect(partitioned[1].fields[0].values.toArray()).toEqual(['C1', 'C2', 'C3']); + expect(partitioned[1].fields[1].values.toArray()).toEqual(['China', 'China', 'China']); + }); + + it('should partition by multiple fields', () => { + const source = [ + toDataFrame({ + name: 'XYZ', + refId: 'A', + fields: [ + { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] }, + { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] }, + { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] }, + ], + }), + ]; + + const config: PartitionByValuesTransformerOptions = { + fields: ['region', 'status'], + }; + + let partitioned = partitionByValuesTransformer.transformer(config)(source); + + expect(partitioned.length).toEqual(4); + + expect(partitioned[0].length).toEqual(1); + expect(partitioned[0].name).toEqual('Europe OK'); + expect(partitioned[0].fields[0].name).toEqual('model'); + expect(partitioned[0].fields[1].name).toEqual('region'); + expect(partitioned[0].fields[2].name).toEqual('status'); + expect(partitioned[0].fields[0].values.toArray()).toEqual(['E1']); + expect(partitioned[0].fields[1].values.toArray()).toEqual(['Europe']); + expect(partitioned[0].fields[2].values.toArray()).toEqual(['OK']); + + expect(partitioned[1].length).toEqual(2); + expect(partitioned[1].name).toEqual('Europe FAIL'); + expect(partitioned[1].fields[0].name).toEqual('model'); + expect(partitioned[1].fields[1].name).toEqual('region'); + expect(partitioned[1].fields[2].name).toEqual('status'); + expect(partitioned[1].fields[0].values.toArray()).toEqual(['E2', 'E3']); + expect(partitioned[1].fields[1].values.toArray()).toEqual(['Europe', 'Europe']); + expect(partitioned[1].fields[2].values.toArray()).toEqual(['FAIL', 'FAIL']); + + expect(partitioned[2].length).toEqual(2); + expect(partitioned[2].name).toEqual('China OK'); + expect(partitioned[2].fields[0].name).toEqual('model'); + expect(partitioned[2].fields[1].name).toEqual('region'); + expect(partitioned[2].fields[2].name).toEqual('status'); + expect(partitioned[2].fields[0].values.toArray()).toEqual(['C1', 'C2']); + expect(partitioned[2].fields[1].values.toArray()).toEqual(['China', 'China']); + expect(partitioned[2].fields[2].values.toArray()).toEqual(['OK', 'OK']); + + expect(partitioned[3].length).toEqual(1); + expect(partitioned[3].name).toEqual('China FAIL'); + expect(partitioned[3].fields[0].name).toEqual('model'); + expect(partitioned[3].fields[1].name).toEqual('region'); + expect(partitioned[3].fields[2].name).toEqual('status'); + expect(partitioned[3].fields[0].values.toArray()).toEqual(['C3']); + expect(partitioned[3].fields[1].values.toArray()).toEqual(['China']); + expect(partitioned[3].fields[2].values.toArray()).toEqual(['FAIL']); + }); + + it('should partition by multiple fields with custom frame naming {withFields: true}', () => { + const source = [ + toDataFrame({ + name: 'XYZ', + refId: 'A', + fields: [ + { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] }, + { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] }, + { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] }, + ], + }), + ]; + + const config: PartitionByValuesTransformerOptions = { + fields: ['region', 'status'], + naming: { + withFields: true, + }, + }; + + let partitioned = partitionByValuesTransformer.transformer(config)(source); + + expect(partitioned[0].name).toEqual('region=Europe status=OK'); + expect(partitioned[1].name).toEqual('region=Europe status=FAIL'); + expect(partitioned[2].name).toEqual('region=China status=OK'); + expect(partitioned[3].name).toEqual('region=China status=FAIL'); + }); + + it('should partition by multiple fields with custom frame naming {append: true}', () => { + const source = [ + toDataFrame({ + name: 'XYZ', + refId: 'A', + fields: [ + { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] }, + { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] }, + { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] }, + ], + }), + ]; + + const config: PartitionByValuesTransformerOptions = { + fields: ['region', 'status'], + naming: { + append: true, + }, + }; + + let partitioned = partitionByValuesTransformer.transformer(config)(source); + + expect(partitioned[0].name).toEqual('XYZ Europe OK'); + expect(partitioned[1].name).toEqual('XYZ Europe FAIL'); + expect(partitioned[2].name).toEqual('XYZ China OK'); + expect(partitioned[3].name).toEqual('XYZ China FAIL'); + }); + + it('should partition by multiple fields with custom frame naming {withFields: true, append: true}', () => { + const source = [ + toDataFrame({ + name: 'XYZ', + refId: 'A', + fields: [ + { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] }, + { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] }, + { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] }, + ], + }), + ]; + + const config: PartitionByValuesTransformerOptions = { + fields: ['region', 'status'], + naming: { + withFields: true, + append: true, + }, + }; + + let partitioned = partitionByValuesTransformer.transformer(config)(source); + + expect(partitioned[0].name).toEqual('XYZ region=Europe status=OK'); + expect(partitioned[1].name).toEqual('XYZ region=Europe status=FAIL'); + expect(partitioned[2].name).toEqual('XYZ region=China status=OK'); + expect(partitioned[3].name).toEqual('XYZ region=China status=FAIL'); + }); +}); diff --git a/public/app/features/transformers/partitionByValues/partitionByValues.ts b/public/app/features/transformers/partitionByValues/partitionByValues.ts new file mode 100644 index 00000000000..645c9882500 --- /dev/null +++ b/public/app/features/transformers/partitionByValues/partitionByValues.ts @@ -0,0 +1,112 @@ +import { map } from 'rxjs'; + +import { + ArrayVector, + DataFrame, + DataTransformerID, + SynchronousDataTransformerInfo, + getFieldMatcher, +} from '@grafana/data'; +import { getMatcherConfig } from '@grafana/data/src/transformations/transformers/filterByName'; +import { noopTransformer } from '@grafana/data/src/transformations/transformers/noop'; + +import { partition } from './partition'; + +export interface FrameNamingOptions { + /** whether to append to existing frame name, false -> replace */ + append?: boolean; // false + /** whether to include discriminator field names, e.g. true -> Region=Europe Profession=Chef, false -> 'Europe Chef' */ + withFields?: boolean; // false + /** name/value separator, e.g. '=' in 'Region=Europe' */ + separator1?: string; + /** name/value pair separator, e.g. ' ' in 'Region=Europe Profession=Chef' */ + separator2?: string; +} + +const defaultFrameNameOptions: FrameNamingOptions = { + append: false, + withFields: false, + separator1: '=', + separator2: ' ', +}; + +export interface PartitionByValuesTransformerOptions { + /** field names whose values should be used as discriminator keys (typically enum fields) */ + fields: string[]; + /** how the split frames' names should be suffixed (ends up as field prefixes) */ + naming?: FrameNamingOptions; +} + +function buildFrameName(opts: FrameNamingOptions, names: string[], values: unknown[]): string { + return names + .map((name, i) => (opts.withFields ? `${name}${opts.separator1}${values[i]}` : values[i])) + .join(opts.separator2); +} + +export const partitionByValuesTransformer: SynchronousDataTransformerInfo = { + id: DataTransformerID.partitionByValues, + name: 'Partition by values', + description: `Splits a one-frame dataset into multiple series discriminated by unique/enum values in one or more fields.`, + defaultOptions: {}, + + operator: (options) => (source) => + source.pipe(map((data) => partitionByValuesTransformer.transformer(options)(data))), + + transformer: (options: PartitionByValuesTransformerOptions) => { + const matcherConfig = getMatcherConfig({ names: options.fields }); + + if (!matcherConfig) { + return noopTransformer.transformer({}); + } + + const matcher = getFieldMatcher(matcherConfig); + + return (data: DataFrame[]) => { + if (!data.length) { + return data; + } + + const frame = data[0]; + const keyFields = frame.fields.filter((f) => matcher!(f, frame, data))!; + const keyFieldsVals = keyFields.map((f) => f.values.toArray()); + const names = keyFields.map((f) => f.name); + + const frameNameOpts = { + ...defaultFrameNameOptions, + ...options.naming, + }; + + return partition(keyFieldsVals).map((idxs: number[]) => { + let name = buildFrameName( + frameNameOpts, + names, + keyFields.map((f, i) => keyFieldsVals[i][idxs[0]]) + ); + + if (options.naming?.append && frame.name) { + name = `${frame.name} ${name}`; + } + + return { + ...frame, + name, + length: idxs.length, + fields: frame.fields.map((f) => { + const vals = f.values.toArray(); + const vals2 = Array(idxs.length); + + for (let i = 0; i < idxs.length; i++) { + vals2[i] = vals[idxs[i]]; + } + + return { + ...f, + state: undefined, + values: new ArrayVector(vals2), + }; + }), + }; + }); + }; + }, +}; diff --git a/public/app/features/transformers/standardTransformers.ts b/public/app/features/transformers/standardTransformers.ts index 99844b9c3cd..c047c59ab14 100644 --- a/public/app/features/transformers/standardTransformers.ts +++ b/public/app/features/transformers/standardTransformers.ts @@ -23,6 +23,7 @@ import { sortByTransformRegistryItem } from './editors/SortByTransformerEditor'; import { extractFieldsTransformRegistryItem } from './extractFields/ExtractFieldsTransformerEditor'; import { joinByLabelsTransformRegistryItem } from './joinByLabels/JoinByLabelsTransformerEditor'; import { fieldLookupTransformRegistryItem } from './lookupGazetteer/FieldLookupTransformerEditor'; +import { partitionByValuesTransformRegistryItem } from './partitionByValues/PartitionByValuesEditor'; import { prepareTimeseriesTransformerRegistryItem } from './prepareTimeSeries/PrepareTimeSeriesEditor'; import { rowsToFieldsTransformRegistryItem } from './rowsToFields/RowsToFieldsTransformerEditor'; import { spatialTransformRegistryItem } from './spatial/SpatialTransformerEditor'; @@ -55,5 +56,6 @@ export const getStandardTransformers = (): Array> = groupingToMatrixTransformRegistryItem, limitTransformRegistryItem, joinByLabelsTransformRegistryItem, + partitionByValuesTransformRegistryItem, ]; };