From 30de9277649d0cfc9ae55e2637eeddf92357556f Mon Sep 17 00:00:00 2001 From: Ben Sully Date: Thu, 2 Dec 2021 08:48:26 +0000 Subject: [PATCH] Add 'JSON' field type to ConvertFieldTypeTransformer I've been playing around with #41994 and found that it requires fields to contain array values, which can't be sent from a backend plugin. This PR adds the ability for the ConvertFieldTypeTransformer to parse JSON values and store the result in the transformed field. The main use case for this right now is so that a field containing a JSONified array can be transformed into a field containing an actual array, which can in turn be used for the table charts in #41994. Supersedes #42521. --- docs/sources/panels/transformations/_index.md | 1 + .../panels/transformations/types-options.md | 30 ++++--- .../transformers/convertFieldType.test.ts | 81 +++++++++++++++++++ .../transformers/convertFieldType.ts | 20 +++++ .../ConvertFieldTypeTransformerEditor.tsx | 1 + 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/docs/sources/panels/transformations/_index.md b/docs/sources/panels/transformations/_index.md index d19e4b5e98a..fe52948e93e 100644 --- a/docs/sources/panels/transformations/_index.md +++ b/docs/sources/panels/transformations/_index.md @@ -29,6 +29,7 @@ Before you can configure and apply transformations: - [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}}) - [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}}) - [Config from query results]({{< relref "./config-from-query.md" >}}) +- [Convert field type]({{< relref "./convert-field-type.md" >}}) - [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}}) - [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}}) - [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}}) diff --git a/docs/sources/panels/transformations/types-options.md b/docs/sources/panels/transformations/types-options.md index d69948f8525..a898d213524 100644 --- a/docs/sources/panels/transformations/types-options.md +++ b/docs/sources/panels/transformations/types-options.md @@ -349,24 +349,28 @@ This transformation changes the field type of the specified field. - **Time -** attempts to parse the values as time - Will show an option to specify a DateFormat for the input field like yyyy-mm-dd or DD MM YYYY hh:mm:ss - **Boolean -** will make the values booleans + - **JSON -** attempts to parse the values as JSON, potentially resulting in complex objects or arrays -For example the following query could be modified by selecting the time field, as Time, and Input format as YYYY. +For example the following query could be modified by: -| Time | Mark | Value | -| ---------- | ----- | ----- | -| 2017-07-01 | above | 25 | -| 2018-08-02 | below | 22 | -| 2019-09-02 | below | 29 | -| 2020-10-04 | above | 22 | +- selecting the time field as Time, and Input format as YYYY +- selecting the JSON field as JSON + +| Time | Mark | Value | JSON | +| ---------- | ----- | ----- | -------- | +| 2017-07-01 | above | 25 | "[0, 1]" | +| 2018-08-02 | below | 22 | "[2, 3]" | +| 2019-09-02 | below | 29 | "[4, 5]" | +| 2020-10-04 | above | 22 | "[6, 7]" | The result: -| Time | Mark | Value | -| ------------------- | ----- | ----- | -| 2017-01-01 00:00:00 | above | 25 | -| 2018-01-01 00:00:00 | below | 22 | -| 2019-01-01 00:00:00 | below | 29 | -| 2020-01-01 00:00:00 | above | 22 | +| Time | Mark | Value | JSON | +| ------------------- | ----- | ----- | ------ | +| 2017-01-01 00:00:00 | above | 25 | [0, 1] | +| 2018-01-01 00:00:00 | below | 22 | [2, 3] | +| 2019-01-01 00:00:00 | below | 29 | [4, 5] | +| 2020-01-01 00:00:00 | above | 22 | [6, 7] | ## Series to rows diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts index 4fdd288a83e..f957732cd4b 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts @@ -183,6 +183,87 @@ describe('field convert types transformer', () => { ]); }); + it('will convert field to complex objects', () => { + const options = { + conversions: [ + { targetField: 'numbers', destinationType: FieldType.other }, + { targetField: 'objects', destinationType: FieldType.other }, + { targetField: 'arrays', destinationType: FieldType.other }, + { targetField: 'invalids', destinationType: FieldType.other }, + { targetField: 'mixed', destinationType: FieldType.other }, + ], + }; + + const comboTypes = toDataFrame({ + fields: [ + { + name: 'numbers', + type: FieldType.number, + values: [-1, 1, null], + }, + { + name: 'objects', + type: FieldType.string, + values: [ + '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }', + '{ "neg": -10, "zero": 0, "pos": 2, "null": null, "array": [2, 3, 4], "nested": { "number": 2 } }', + '{ "neg": -1, "zero": 0, "pos": 3, "null": null, "array": [3, 4, 5], "nested": { "number": 3 } }', + ], + }, + { + name: 'arrays', + type: FieldType.string, + values: ['[true]', '[99]', '["2021-08-02 00:00:00.000"]'], + }, + { + name: 'invalids', + type: FieldType.string, + values: ['abcd', '{ invalidJson }', '[unclosed array'], + }, + { + name: 'mixed', + type: FieldType.string, + values: [ + '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }', + '["a string", 1234, {"a complex": "object"}]', + '["this is invalid JSON]', + ], + }, + ], + }); + + const complex = convertFieldTypes(options, [comboTypes]); + expect( + complex[0].fields.map((f) => ({ + type: f.type, + values: f.values.toArray(), + })) + ).toEqual([ + { + type: FieldType.other, + values: [-1, 1, null], + }, + { + type: FieldType.other, + values: [ + { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } }, + { neg: -10, zero: 0, pos: 2, null: null, array: [2, 3, 4], nested: { number: 2 } }, + { neg: -1, zero: 0, pos: 3, null: null, array: [3, 4, 5], nested: { number: 3 } }, + ], + }, + { type: FieldType.other, values: [[true], [99], ['2021-08-02 00:00:00.000']] }, + { type: FieldType.other, values: [null, null, null] }, + { + type: FieldType.other, + values: [ + { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } }, + ['a string', 1234, { 'a complex': 'object' }], + null, + ], + }, + ]); + }); + it('will convert field to strings', () => { const options = { conversions: [{ targetField: 'numbers', destinationType: FieldType.string }], diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts index 7eeeef27276..967bdccaa38 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts @@ -99,6 +99,8 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F return fieldToStringField(field); case FieldType.boolean: return fieldToBooleanField(field); + case FieldType.other: + return fieldToComplexField(field); default: return field; } @@ -178,6 +180,24 @@ function fieldToStringField(field: Field): Field { }; } +function fieldToComplexField(field: Field): Field { + const complexValues = field.values.toArray().slice(); + + for (let s = 0; s < complexValues.length; s++) { + try { + complexValues[s] = JSON.parse(complexValues[s]); + } catch { + complexValues[s] = null; + } + } + + return { + ...field, + type: FieldType.other, + values: new ArrayVector(complexValues), + }; +} + /** * Checks the first value. Assumes any number should be time fieldtype. Otherwise attempts to make the fieldtype time. * @param field - field to ensure is a time fieldtype diff --git a/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx index 6aaa2777239..456761e5c5c 100644 --- a/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx +++ b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx @@ -29,6 +29,7 @@ export const ConvertFieldTypeTransformerEditor: React.FC