From 13d9fddd3e18e54ae5a84d441ae1bdd9e6c1e71a Mon Sep 17 00:00:00 2001 From: Ben Sully Date: Thu, 16 Dec 2021 10:18:11 +0000 Subject: [PATCH] Transformations: Add 'JSON' field type to ConvertFieldTypeTransformer (#42624) * Add 'JSON' field type to ConvertFieldTypeTransformer I've been playing around with #41994 and found that it requires fields to contain array values, which can't be sent from a backend plugin. This PR adds the ability for the ConvertFieldTypeTransformer to parse JSON values and store the result in the transformed field. The main use case for this right now is so that a field containing a JSONified array can be transformed into a field containing an actual array, which can in turn be used for the table charts in #41994. Supersedes #42521. * Add second option to complex field conversion to increase flexibility This avoids falsely equating 'JSON' with FieldType.other, and instead allows multiple parsers to be used if the 'Complex' type is selected. Currently only JSON parsing is implemented, but others could be supported easily in future. * Revert "Add second option to complex field conversion to increase flexibility" This reverts commit 6314ce35eb6d8136c8e238abd2279cfa50ef3cc2. * Improve test for object parsing of complex field transformer Co-authored-by: Ryan McKinley --- docs/sources/panels/transformations/_index.md | 1 + .../panels/transformations/types-options.md | 30 ++++--- .../transformers/convertFieldType.test.ts | 81 +++++++++++++++++++ .../transformers/convertFieldType.ts | 20 +++++ .../ConvertFieldTypeTransformerEditor.tsx | 1 + 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/docs/sources/panels/transformations/_index.md b/docs/sources/panels/transformations/_index.md index d19e4b5e98a..fe52948e93e 100644 --- a/docs/sources/panels/transformations/_index.md +++ b/docs/sources/panels/transformations/_index.md @@ -29,6 +29,7 @@ Before you can configure and apply transformations: - [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}}) - [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}}) - [Config from query results]({{< relref "./config-from-query.md" >}}) +- [Convert field type]({{< relref "./convert-field-type.md" >}}) - [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}}) - [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}}) - [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}}) diff --git a/docs/sources/panels/transformations/types-options.md b/docs/sources/panels/transformations/types-options.md index d69948f8525..a898d213524 100644 --- a/docs/sources/panels/transformations/types-options.md +++ b/docs/sources/panels/transformations/types-options.md @@ -349,24 +349,28 @@ This transformation changes the field type of the specified field. - **Time -** attempts to parse the values as time - Will show an option to specify a DateFormat for the input field like yyyy-mm-dd or DD MM YYYY hh:mm:ss - **Boolean -** will make the values booleans + - **JSON -** attempts to parse the values as JSON, potentially resulting in complex objects or arrays -For example the following query could be modified by selecting the time field, as Time, and Input format as YYYY. +For example the following query could be modified by: -| Time | Mark | Value | -| ---------- | ----- | ----- | -| 2017-07-01 | above | 25 | -| 2018-08-02 | below | 22 | -| 2019-09-02 | below | 29 | -| 2020-10-04 | above | 22 | +- selecting the time field as Time, and Input format as YYYY +- selecting the JSON field as JSON + +| Time | Mark | Value | JSON | +| ---------- | ----- | ----- | -------- | +| 2017-07-01 | above | 25 | "[0, 1]" | +| 2018-08-02 | below | 22 | "[2, 3]" | +| 2019-09-02 | below | 29 | "[4, 5]" | +| 2020-10-04 | above | 22 | "[6, 7]" | The result: -| Time | Mark | Value | -| ------------------- | ----- | ----- | -| 2017-01-01 00:00:00 | above | 25 | -| 2018-01-01 00:00:00 | below | 22 | -| 2019-01-01 00:00:00 | below | 29 | -| 2020-01-01 00:00:00 | above | 22 | +| Time | Mark | Value | JSON | +| ------------------- | ----- | ----- | ------ | +| 2017-01-01 00:00:00 | above | 25 | [0, 1] | +| 2018-01-01 00:00:00 | below | 22 | [2, 3] | +| 2019-01-01 00:00:00 | below | 29 | [4, 5] | +| 2020-01-01 00:00:00 | above | 22 | [6, 7] | ## Series to rows diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts index 4fdd288a83e..8c74f1edbcd 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts @@ -183,6 +183,87 @@ describe('field convert types transformer', () => { ]); }); + it('will convert field to complex objects', () => { + const options = { + conversions: [ + { targetField: 'numbers', destinationType: FieldType.other }, + { targetField: 'objects', destinationType: FieldType.other }, + { targetField: 'arrays', destinationType: FieldType.other }, + { targetField: 'invalids', destinationType: FieldType.other }, + { targetField: 'mixed', destinationType: FieldType.other }, + ], + }; + + const comboTypes = toDataFrame({ + fields: [ + { + name: 'numbers', + type: FieldType.number, + values: [-1, 1, null], + }, + { + name: 'objects', + type: FieldType.string, + values: [ + '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }', + '{ "string": "abcd" }', + '{}', + ], + }, + { + name: 'arrays', + type: FieldType.string, + values: ['[true]', '[99]', '["2021-08-02 00:00:00.000"]'], + }, + { + name: 'invalids', + type: FieldType.string, + values: ['abcd', '{ invalidJson }', '[unclosed array'], + }, + { + name: 'mixed', + type: FieldType.string, + values: [ + '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }', + '["a string", 1234, {"a complex": "object"}]', + '["this is invalid JSON]', + ], + }, + ], + }); + + const complex = convertFieldTypes(options, [comboTypes]); + expect( + complex[0].fields.map((f) => ({ + type: f.type, + values: f.values.toArray(), + })) + ).toEqual([ + { + type: FieldType.other, + values: [-1, 1, null], + }, + { + type: FieldType.other, + values: [ + { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } }, + { string: 'abcd' }, + {}, + ], + }, + { type: FieldType.other, values: [[true], [99], ['2021-08-02 00:00:00.000']] }, + { type: FieldType.other, values: [null, null, null] }, + { + type: FieldType.other, + values: [ + { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } }, + ['a string', 1234, { 'a complex': 'object' }], + null, + ], + }, + ]); + }); + it('will convert field to strings', () => { const options = { conversions: [{ targetField: 'numbers', destinationType: FieldType.string }], diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts index 7eeeef27276..967bdccaa38 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts @@ -99,6 +99,8 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F return fieldToStringField(field); case FieldType.boolean: return fieldToBooleanField(field); + case FieldType.other: + return fieldToComplexField(field); default: return field; } @@ -178,6 +180,24 @@ function fieldToStringField(field: Field): Field { }; } +function fieldToComplexField(field: Field): Field { + const complexValues = field.values.toArray().slice(); + + for (let s = 0; s < complexValues.length; s++) { + try { + complexValues[s] = JSON.parse(complexValues[s]); + } catch { + complexValues[s] = null; + } + } + + return { + ...field, + type: FieldType.other, + values: new ArrayVector(complexValues), + }; +} + /** * Checks the first value. Assumes any number should be time fieldtype. Otherwise attempts to make the fieldtype time. * @param field - field to ensure is a time fieldtype diff --git a/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx index 6aaa2777239..456761e5c5c 100644 --- a/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx +++ b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx @@ -29,6 +29,7 @@ export const ConvertFieldTypeTransformerEditor: React.FC