From b312a60627e7bdb53a7466da6cc6d7ca7062c1e0 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Thu, 29 Jul 2021 22:25:04 +0100 Subject: [PATCH] Transformers: allow label fields in the reduce processor (#37373) (#37379) Co-authored-by: Ryan McKinley (cherry picked from commit c564736c687367254ce69932ab37539132a7273f) Co-authored-by: An --- .../transformers/reduce.test.ts | 84 +++++++++++++- .../transformations/transformers/reduce.ts | 103 ++++++++++++------ .../ReduceTransformerEditor.tsx | 19 ++++ 3 files changed, 171 insertions(+), 35 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformers/reduce.test.ts b/packages/grafana-data/src/transformations/transformers/reduce.test.ts index b14d1d534ac..193589c4479 100644 --- a/packages/grafana-data/src/transformations/transformers/reduce.test.ts +++ b/packages/grafana-data/src/transformations/transformers/reduce.test.ts @@ -2,9 +2,9 @@ import { ReducerID } from '../fieldReducer'; import { DataTransformerID } from './ids'; import { toDataFrame } from '../../dataframe/processDataFrame'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; -import { reduceFields, reduceTransformer } from './reduce'; +import { reduceFields, reduceTransformer, ReduceTransformerOptions } from './reduce'; import { transformDataFrame } from '../transformDataFrame'; -import { Field, FieldType } from '../../types'; +import { DataTransformerConfig, Field, FieldType } from '../../types'; import { ArrayVector } from '../../vector'; import { notTimeFieldMatcher } from '../matchers/predicates'; import { DataFrameView } from '../../dataframe'; @@ -366,4 +366,84 @@ describe('Reducer Transformer', () => { expect(processed[0].fields).toEqual(expected); }); }); + + it('reduces keeping label field', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.reduce, + options: { + reducers: [ReducerID.max], + labelsToFields: true, + }, + }; + + const seriesA = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, + { name: 'value', labels: { state: 'CA' }, type: FieldType.number, values: [3, 4, 5, 6] }, + { name: 'value', labels: { state: 'NY' }, type: FieldType.number, values: [3, 4, 5, 6] }, + ], + }); + + const seriesB = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, + { name: 'value', labels: { state: 'CA', country: 'USA' }, type: FieldType.number, values: [3, 4, 5, 6] }, + { name: 'value', labels: { country: 'USA' }, type: FieldType.number, values: [3, 4, 5, 6] }, + ], + }); + + await expect(transformDataFrame([cfg], [seriesA, seriesB])).toEmitValuesWith((received) => { + const processed = received[0]; + + expect(processed.length).toEqual(1); + expect(processed[0].fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "Field", + "type": "string", + "values": Array [ + "value", + "value", + "value", + "value", + ], + }, + Object { + "config": Object {}, + "name": "state", + "type": "string", + "values": Array [ + "CA", + "NY", + "CA", + undefined, + ], + }, + Object { + "config": Object {}, + "name": "country", + "type": "string", + "values": Array [ + undefined, + undefined, + "USA", + "USA", + ], + }, + Object { + "config": Object {}, + "name": "Max", + "type": "number", + "values": Array [ + 6, + 6, + 6, + 6, + ], + }, + ] + `); + }); + }); }); diff --git a/packages/grafana-data/src/transformations/transformers/reduce.ts b/packages/grafana-data/src/transformations/transformers/reduce.ts index 413f6c561e3..ec229e60a7d 100644 --- a/packages/grafana-data/src/transformations/transformers/reduce.ts +++ b/packages/grafana-data/src/transformations/transformers/reduce.ts @@ -20,6 +20,7 @@ export interface ReduceTransformerOptions { fields?: MatcherConfig; // Assume all fields mode?: ReduceTransformerMode; includeTimeField?: boolean; + labelsToFields?: boolean; } export const reduceTransformer: DataTransformerInfo = { @@ -53,7 +54,7 @@ export const reduceTransformer: DataTransformerInfo = } // Add a row for each series - const res = reduceSeriesToRows(data, matcher, options.reducers); + const res = reduceSeriesToRows(data, matcher, options.reducers, options.labelsToFields); return res ? [res] : []; }) ), @@ -65,77 +66,113 @@ export const reduceTransformer: DataTransformerInfo = export function reduceSeriesToRows( data: DataFrame[], matcher: FieldMatcher, - reducerId: ReducerID[] + reducerId: ReducerID[], + labelsToFields?: boolean ): DataFrame | undefined { const calculators = fieldReducers.list(reducerId); const reducers = calculators.map((c) => c.id); const processed: DataFrame[] = []; + const distinctLabels = labelsToFields ? getDistinctLabelKeys(data) : []; for (const series of data) { - const values: ArrayVector[] = []; - const fields: Field[] = []; - const byId: KeyValue = {}; + const source = series.fields.filter((f) => matcher(f, series, data)); - values.push(new ArrayVector()); // The name + const size = source.length; + const fields: Field[] = []; + const names = new ArrayVector(new Array(size)); fields.push({ name: 'Field', type: FieldType.string, - values: values[0], + values: names, config: {}, }); - for (const info of calculators) { - const vals = new ArrayVector(); - byId[info.id] = vals; - values.push(vals); + const labels: KeyValue = {}; + if (labelsToFields) { + for (const key of distinctLabels) { + labels[key] = new ArrayVector(new Array(size)); + fields.push({ + name: key, + type: FieldType.string, + values: labels[key], + config: {}, + }); + } + } + const calcs: KeyValue = {}; + for (const info of calculators) { + calcs[info.id] = new ArrayVector(new Array(size)); fields.push({ name: info.name, type: FieldType.other, // UNKNOWN until after we call the functions - values: values[values.length - 1], + values: calcs[info.id], config: {}, }); } - for (let i = 0; i < series.fields.length; i++) { - const field = series.fields[i]; + for (let i = 0; i < source.length; i++) { + const field = source[i]; + const results = reduceField({ + field, + reducers, + }); - if (matcher(field, series, data)) { - const results = reduceField({ - field, - reducers, - }); - - // Update the name list - const fieldName = getFieldDisplayName(field, series, data); - - values[0].buffer.push(fieldName); - - for (const info of calculators) { - const v = results[info.id]; - byId[info.id].buffer.push(v); + if (labelsToFields) { + names.buffer[i] = field.name; + if (field.labels) { + for (const key of Object.keys(field.labels)) { + labels[key].set(i, field.labels[key]); + } } + } else { + names.buffer[i] = getFieldDisplayName(field, series, data); + } + + for (const info of calculators) { + const v = results[info.id]; + calcs[info.id].buffer[i] = v; } } + // For reduced fields, we don't know the type until we see the value for (const f of fields) { - const t = guessFieldTypeForField(f); - - if (t) { - f.type = t; + if (f.type === FieldType.other) { + const t = guessFieldTypeForField(f); + if (t) { + f.type = t; + } } } processed.push({ ...series, // Same properties, different fields fields, - length: values[0].length, + length: size, }); } return mergeResults(processed); } +export function getDistinctLabelKeys(frames: DataFrame[]): string[] { + const ordered: string[] = []; + const keys = new Set(); + for (const frame of frames) { + for (const field of frame.fields) { + if (field.labels) { + for (const k of Object.keys(field.labels)) { + if (!keys.has(k)) { + ordered.push(k); + keys.add(k); + } + } + } + } + } + return ordered; +} + /** * @internal only exported for testing */ diff --git a/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx b/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx index 81da6b5bf3e..93fbc712c4b 100644 --- a/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx +++ b/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx @@ -49,6 +49,13 @@ export const ReduceTransformerEditor: React.FC { + onChange({ + ...options, + labelsToFields: !options.labelsToFields, + }); + }, [onChange, options]); + return ( <>
@@ -95,6 +102,18 @@ export const ReduceTransformerEditor: React.FC
)} + {options.mode !== ReduceTransformerMode.ReduceFields && ( +
+
+ +
+
+ )} ); };