From 229176f1b06918f996d3b1c5f20885a7473e3cd1 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 20 Apr 2020 09:57:04 -0700 Subject: [PATCH] Transformers: calculate a new field based on the row values (#23675) --- .../src/transformations/fieldReducer.ts | 2 +- .../grafana-data/src/transformations/index.ts | 1 + .../src/transformations/transformers.ts | 2 + .../transformers/calculateField.test.ts | 101 +++++++++ .../transformers/calculateField.ts | 82 +++++++ .../src/transformations/transformers/ids.ts | 1 + packages/grafana-data/src/vector/RowVector.ts | 28 +++ .../CalculateFieldTransformerEditor.tsx | 203 ++++++++++++++++++ .../src/utils/standardTransformers.ts | 2 + 9 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 packages/grafana-data/src/transformations/transformers/calculateField.test.ts create mode 100644 packages/grafana-data/src/transformations/transformers/calculateField.ts create mode 100644 packages/grafana-data/src/vector/RowVector.ts create mode 100644 packages/grafana-ui/src/components/TransformersUI/CalculateFieldTransformerEditor.tsx diff --git a/packages/grafana-data/src/transformations/fieldReducer.ts b/packages/grafana-data/src/transformations/fieldReducer.ts index 06614d4c511..e15a5f0a329 100644 --- a/packages/grafana-data/src/transformations/fieldReducer.ts +++ b/packages/grafana-data/src/transformations/fieldReducer.ts @@ -227,7 +227,7 @@ export const fieldReducers = new Registry(() => [ }, ]); -function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs { +export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs { const calcs = { sum: 0, max: -Number.MAX_VALUE, diff --git a/packages/grafana-data/src/transformations/index.ts b/packages/grafana-data/src/transformations/index.ts index 6e294edafa0..1a4beed28d3 100644 --- a/packages/grafana-data/src/transformations/index.ts +++ b/packages/grafana-data/src/transformations/index.ts @@ -7,6 +7,7 @@ export { FilterFieldsByNameTransformerOptions } from './transformers/filterByNam export { FilterFramesByRefIdTransformerOptions } from './transformers/filterByRefId'; export { SeriesToColumnsOptions } from './transformers/seriesToColumns'; export { ReduceTransformerOptions } from './transformers/reduce'; +export { CalculateFieldTransformerOptions } from './transformers/calculateField'; export { OrganizeFieldsTransformerOptions } from './transformers/organize'; export { createOrderFieldsComparer } from './transformers/order'; export { transformDataFrame } from './transformDataFrame'; diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index f26826b517a..d7e746fec45 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -1,5 +1,6 @@ import { appendTransformer } from './transformers/append'; import { reduceTransformer } from './transformers/reduce'; +import { calculateFieldTransformer } from './transformers/calculateField'; import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter'; import { filterFieldsByNameTransformer } from './transformers/filterByName'; import { noopTransformer } from './transformers/noop'; @@ -19,6 +20,7 @@ export const standardTransformers = { organizeFieldsTransformer, appendTransformer, reduceTransformer, + calculateFieldTransformer, seriesToColumnsTransformer, renameFieldsTransformer, }; diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.test.ts b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts new file mode 100644 index 00000000000..e51f56bbfa0 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts @@ -0,0 +1,101 @@ +import { DataTransformerID } from './ids'; +import { toDataFrame } from '../../dataframe/processDataFrame'; +import { FieldType } from '../../types/dataFrame'; +import { ReducerID } from '../fieldReducer'; +import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { transformDataFrame } from '../transformDataFrame'; +import { calculateFieldTransformer } from './calculateField'; +import { DataFrameView } from '../../dataframe'; + +const seriesToTestWith = toDataFrame({ + fields: [ + { name: 'A', type: FieldType.time, values: [1000, 2000] }, + { name: 'B', type: FieldType.number, values: [1, 100] }, + { name: 'C', type: FieldType.number, values: [2, 200] }, + { name: 'D', type: FieldType.string, values: ['first', 'second'] }, + ], +}); + +describe('calculateField transformer', () => { + beforeAll(() => { + mockTransformationsRegistry([calculateFieldTransformer]); + }); + + it('will filter and alias', () => { + const cfg = { + id: DataTransformerID.calculateField, + options: { + // defautls to sum + alias: 'The Total', + }, + }; + + const filtered = transformDataFrame([cfg], [seriesToTestWith])[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toMatchInlineSnapshot(` + Array [ + Object { + "A": 1000, + "B": 1, + "C": 2, + "D": "first", + "The Total": 3, + }, + Object { + "A": 2000, + "B": 100, + "C": 200, + "D": "second", + "The Total": 300, + }, + ] + `); + }); + + it('will replace other fields', () => { + const cfg = { + id: DataTransformerID.calculateField, + options: { + reducer: ReducerID.mean, + replaceFields: true, + }, + }; + + const filtered = transformDataFrame([cfg], [seriesToTestWith])[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toMatchInlineSnapshot(` + Array [ + Object { + "Mean": 1.5, + }, + Object { + "Mean": 150, + }, + ] + `); + }); + + it('will filter by name', () => { + const cfg = { + id: DataTransformerID.calculateField, + options: { + reducer: ReducerID.mean, + replaceFields: true, + include: 'B', + }, + }; + + const filtered = transformDataFrame([cfg], [seriesToTestWith])[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toMatchInlineSnapshot(` + Array [ + Object { + "Mean": 1, + }, + Object { + "Mean": 100, + }, + ] + `); + }); +}); diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.ts b/packages/grafana-data/src/transformations/transformers/calculateField.ts new file mode 100644 index 00000000000..68f030932ec --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/calculateField.ts @@ -0,0 +1,82 @@ +import { DataFrame, DataTransformerInfo, Vector, FieldType, Field, NullValueMode } from '../../types'; +import { DataTransformerID } from './ids'; +import { ReducerID, fieldReducers } from '../fieldReducer'; +import { getFieldMatcher } from '../matchers'; +import { FieldMatcherID } from '../matchers/ids'; +import { RowVector } from '../../vector/RowVector'; +import { ArrayVector } from '../../vector'; +import { doStandardCalcs } from '../fieldReducer'; + +export interface CalculateFieldTransformerOptions { + reducer: ReducerID; + include?: string; // Assume all fields + alias?: string; // The output field name + replaceFields?: boolean; + nullValueMode?: NullValueMode; +} + +export const calculateFieldTransformer: DataTransformerInfo = { + id: DataTransformerID.calculateField, + name: 'Add field from calculation', + description: 'Use the row values to calculate a new field', + defaultOptions: { + reducer: ReducerID.sum, + }, + transformer: options => (data: DataFrame[]) => { + let matcher = getFieldMatcher({ + id: FieldMatcherID.numeric, + }); + if (options.include && options.include.length) { + matcher = getFieldMatcher({ + id: FieldMatcherID.byName, + options: options.include, + }); + } + + const info = fieldReducers.get(options.reducer); + if (!info) { + throw new Error(`Unknown reducer: ${options.reducer}`); + } + const reducer = info.reduce ?? doStandardCalcs; + const ignoreNulls = options.nullValueMode === NullValueMode.Ignore; + const nullAsZero = options.nullValueMode === NullValueMode.AsZero; + + return data.map(frame => { + // Find the columns that should be examined + const columns: Vector[] = []; + frame.fields.forEach(field => { + if (matcher(field)) { + columns.push(field.values); + } + }); + + // Prepare a "fake" field for the row + const iter = new RowVector(columns); + const row: Field = { + name: 'temp', + values: iter, + type: FieldType.number, + config: {}, + }; + const vals: number[] = []; + for (let i = 0; i < frame.length; i++) { + iter.rowIndex = i; + row.calcs = undefined; // bust the cache (just in case) + const val = reducer(row, ignoreNulls, nullAsZero)[options.reducer]; + vals.push(val); + } + + const field = { + name: options.alias || info.name, + type: FieldType.number, + config: {}, + values: new ArrayVector(vals), + }; + + return { + ...frame, + fields: options.replaceFields ? [field] : [...frame.fields, field], + }; + }); + }, +}; diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index 686cbd847ef..61ab863abb0 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -6,6 +6,7 @@ export enum DataTransformerID { order = 'order', // order fields based on user configuration organize = 'organize', // order, rename and filter based on user configuration rename = 'rename', // rename field based on user configuration + calculateField = 'calculateField', // Run a reducer on the row seriesToColumns = 'seriesToColumns', // former table transform timeseries_to_columns filterFields = 'filterFields', // Pick some fields (keep all frames) diff --git a/packages/grafana-data/src/vector/RowVector.ts b/packages/grafana-data/src/vector/RowVector.ts new file mode 100644 index 00000000000..d05904c28b3 --- /dev/null +++ b/packages/grafana-data/src/vector/RowVector.ts @@ -0,0 +1,28 @@ +import { Vector } from '../types'; +import { vectorToArray } from './vectorToArray'; + +/** + * RowVector makes the row values look like a vector + * @internal + */ +export class RowVector implements Vector { + constructor(private columns: Vector[]) {} + + rowIndex = 0; + + get length(): number { + return this.columns.length; + } + + get(index: number): number { + return this.columns[index].get(this.rowIndex); + } + + toArray(): number[] { + return vectorToArray(this); + } + + toJSON(): number[] { + return vectorToArray(this); + } +} diff --git a/packages/grafana-ui/src/components/TransformersUI/CalculateFieldTransformerEditor.tsx b/packages/grafana-ui/src/components/TransformersUI/CalculateFieldTransformerEditor.tsx new file mode 100644 index 00000000000..3afa5885ab8 --- /dev/null +++ b/packages/grafana-ui/src/components/TransformersUI/CalculateFieldTransformerEditor.tsx @@ -0,0 +1,203 @@ +import React, { useContext, ChangeEvent } from 'react'; +import { + DataTransformerID, + CalculateFieldTransformerOptions, + KeyValue, + standardTransformers, + TransformerRegistyItem, + TransformerUIProps, + FieldType, + ReducerID, + fieldReducers, +} from '@grafana/data'; +import { ThemeContext } from '../../themes/ThemeContext'; +import { css } from 'emotion'; +import { InlineList } from '../List/InlineList'; +import { Icon } from '../Icon/Icon'; +import { Label } from '../Forms/Label'; +import { StatsPicker } from '../StatsPicker/StatsPicker'; +import { Switch } from '../Switch/Switch'; +import { Input } from '../Input/Input'; + +interface CalculateFieldTransformerEditorProps extends TransformerUIProps {} + +interface CalculateFieldTransformerEditorState { + include: string; + names: string[]; + selected: string[]; +} + +export class CalculateFieldTransformerEditor extends React.PureComponent< + CalculateFieldTransformerEditorProps, + CalculateFieldTransformerEditorState +> { + constructor(props: CalculateFieldTransformerEditorProps) { + super(props); + this.state = { + include: props.options.include || '', + names: [], + selected: [], + }; + } + + componentDidMount() { + this.initOptions(); + } + + private initOptions() { + const { input, options } = this.props; + const configuredOptions = options.include ? options.include.split('|') : []; + + const allNames: string[] = []; + const byName: KeyValue = {}; + for (const frame of input) { + for (const field of frame.fields) { + if (field.type !== FieldType.number) { + continue; + } + if (!byName[field.name]) { + byName[field.name] = true; + allNames.push(field.name); + } + } + } + + if (configuredOptions.length) { + const options: string[] = []; + const selected: string[] = []; + for (const v of allNames) { + if (configuredOptions.includes(v)) { + selected.push(v); + } + options.push(v); + } + + this.setState({ + names: options, + selected: selected, + }); + } else { + this.setState({ names: allNames, selected: [] }); + } + } + + onFieldToggle = (fieldName: string) => { + const { selected } = this.state; + if (selected.indexOf(fieldName) > -1) { + this.onChange(selected.filter(s => s !== fieldName)); + } else { + this.onChange([...selected, fieldName]); + } + }; + + onChange = (selected: string[]) => { + this.setState({ selected }); + this.props.onChange({ + ...this.props.options, + include: selected.join('|'), + }); + }; + + onToggleReplaceFields = (evt: ChangeEvent) => { + const { options } = this.props; + this.props.onChange({ + ...options, + replaceFields: !options.replaceFields, + }); + }; + + onAliasChanged = (evt: ChangeEvent) => { + const { options } = this.props; + this.props.onChange({ + ...options, + alias: evt.target.value, + }); + }; + + onStatsChange = (stats: string[]) => { + this.props.onChange({ + ...this.props.options, + reducer: stats.length ? (stats[0] as ReducerID) : ReducerID.sum, + }); + }; + + render() { + const { options } = this.props; + const { names, selected } = this.state; + const reducer = fieldReducers.get(options.reducer); + return ( +
+ + { + return ( + + { + this.onFieldToggle(o); + }} + label={o} + selected={selected.indexOf(o) > -1} + /> + + ); + }} + /> + + + + + + + + + {/* nullValueMode?: NullValueMode; */} +
+ ); + } +} + +interface FilterPillProps { + selected: boolean; + label: string; + onClick: React.MouseEventHandler; +} +const FilterPill: React.FC = ({ label, selected, onClick }) => { + const theme = useContext(ThemeContext); + return ( +
+ {selected && ( + + )} + {label} +
+ ); +}; + +export const calculateFieldTransformRegistryItem: TransformerRegistyItem = { + id: DataTransformerID.calculateField, + editor: CalculateFieldTransformerEditor, + transformation: standardTransformers.calculateFieldTransformer, + name: 'Add field from calculation', + description: 'Use the row values to calculate a new field', +}; diff --git a/packages/grafana-ui/src/utils/standardTransformers.ts b/packages/grafana-ui/src/utils/standardTransformers.ts index fc45ba13ab9..914f82aa86c 100644 --- a/packages/grafana-ui/src/utils/standardTransformers.ts +++ b/packages/grafana-ui/src/utils/standardTransformers.ts @@ -4,6 +4,7 @@ import { filterFieldsByNameTransformRegistryItem } from '../components/Transform import { filterFramesByRefIdTransformRegistryItem } from '../components/TransformersUI/FilterByRefIdTransformerEditor'; import { organizeFieldsTransformRegistryItem } from '../components/TransformersUI/OrganizeFieldsTransformerEditor'; import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor'; +import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -12,5 +13,6 @@ export const getStandardTransformers = (): Array> => filterFramesByRefIdTransformRegistryItem, organizeFieldsTransformRegistryItem, seriesToFieldsTransformerRegistryItem, + calculateFieldTransformRegistryItem, ]; };