public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 'JSON' field type to ConvertFieldTypeTransformer

Browse Source 
I've been playing around with #41994 and found that it requires fields
to contain array values, which can't be sent from a backend plugin. This
PR adds the ability for the ConvertFieldTypeTransformer to parse
JSON values and store the result in the transformed field.
The main use case for this right now is so that a field
containing a JSONified array can be transformed into a field
containing an actual array, which can in
turn be used for the table charts in #41994.

Supersedes #42521.
This commit is contained in:
Ben Sully
2021-12-02 12:28:55 +00:00
parent 54fa7b57fe
commit 30de927764
5 changed files with 120 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sources/panels/transformations/_index.md
+1
View File
@@ -29,6 +29,7 @@ Before you can configure and apply transformations:
- [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}})
- [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}})
- [Config from query results]({{< relref "./config-from-query.md" >}})
- [Convert field type]({{< relref "./convert-field-type.md" >}})
- [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}})
- [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}})
- [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}})
docs/sources/panels/transformations/types-options.md
+17 -13
View File
@@ -349,24 +349,28 @@ This transformation changes the field type of the specified field.
- **Time -** attempts to parse the values as time
- Will show an option to specify a DateFormat for the input field like yyyy-mm-dd or DD MM YYYY hh:mm:ss
- **Boolean -** will make the values booleans
- **JSON -** attempts to parse the values as JSON, potentially resulting in complex objects or arrays
For example the following query could be modified by selecting the time field, as Time, and Input format as YYYY.
For example the following query could be modified by:
| Time | Mark | Value |
| ---------- | ----- | ----- |
| 2017-07-01 | above | 25 |
| 2018-08-02 | below | 22 |
| 2019-09-02 | below | 29 |
| 2020-10-04 | above | 22 |
- selecting the time field as Time, and Input format as YYYY
- selecting the JSON field as JSON
| Time | Mark | Value | JSON |
| ---------- | ----- | ----- | -------- |
| 2017-07-01 | above | 25 | "[0, 1]" |
| 2018-08-02 | below | 22 | "[2, 3]" |
| 2019-09-02 | below | 29 | "[4, 5]" |
| 2020-10-04 | above | 22 | "[6, 7]" |
The result:
| Time | Mark | Value |
| ------------------- | ----- | ----- |
| 2017-01-01 00:00:00 | above | 25 |
| 2018-01-01 00:00:00 | below | 22 |
| 2019-01-01 00:00:00 | below | 29 |
| 2020-01-01 00:00:00 | above | 22 |
| Time | Mark | Value | JSON |
| ------------------- | ----- | ----- | ------ |
| 2017-01-01 00:00:00 | above | 25 | [0, 1] |
| 2018-01-01 00:00:00 | below | 22 | [2, 3] |
| 2019-01-01 00:00:00 | below | 29 | [4, 5] |
| 2020-01-01 00:00:00 | above | 22 | [6, 7] |
## Series to rows
packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts
+81
View File
@@ -183,6 +183,87 @@ describe('field convert types transformer', () => {
]);
});
it('will convert field to complex objects', () => {
const options = {
conversions: [
{ targetField: 'numbers', destinationType: FieldType.other },
{ targetField: 'objects', destinationType: FieldType.other },
{ targetField: 'arrays', destinationType: FieldType.other },
{ targetField: 'invalids', destinationType: FieldType.other },
{ targetField: 'mixed', destinationType: FieldType.other },
],
};
const comboTypes = toDataFrame({
fields: [
{
name: 'numbers',
type: FieldType.number,
values: [-1, 1, null],
},
{
name: 'objects',
type: FieldType.string,
values: [
'{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }',
'{ "neg": -10, "zero": 0, "pos": 2, "null": null, "array": [2, 3, 4], "nested": { "number": 2 } }',
'{ "neg": -1, "zero": 0, "pos": 3, "null": null, "array": [3, 4, 5], "nested": { "number": 3 } }',
],
},
{
name: 'arrays',
type: FieldType.string,
values: ['[true]', '[99]', '["2021-08-02 00:00:00.000"]'],
},
{
name: 'invalids',
type: FieldType.string,
values: ['abcd', '{ invalidJson }', '[unclosed array'],
},
{
name: 'mixed',
type: FieldType.string,
values: [
'{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }',
'["a string", 1234, {"a complex": "object"}]',
'["this is invalid JSON]',
],
},
],
});
const complex = convertFieldTypes(options, [comboTypes]);
expect(
complex[0].fields.map((f) => ({
type: f.type,
values: f.values.toArray(),
}))
).toEqual([
{
type: FieldType.other,
values: [-1, 1, null],
},
{
type: FieldType.other,
values: [
{ neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } },
{ neg: -10, zero: 0, pos: 2, null: null, array: [2, 3, 4], nested: { number: 2 } },
{ neg: -1, zero: 0, pos: 3, null: null, array: [3, 4, 5], nested: { number: 3 } },
],
},
{ type: FieldType.other, values: [[true], [99], ['2021-08-02 00:00:00.000']] },
{ type: FieldType.other, values: [null, null, null] },
{
type: FieldType.other,
values: [
{ neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } },
['a string', 1234, { 'a complex': 'object' }],
null,
],
},
]);
});
it('will convert field to strings', () => {
const options = {
conversions: [{ targetField: 'numbers', destinationType: FieldType.string }],
packages/grafana-data/src/transformations/transformers/convertFieldType.ts
+20
View File
@@ -99,6 +99,8 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F
return fieldToStringField(field);
case FieldType.boolean:
return fieldToBooleanField(field);
case FieldType.other:
return fieldToComplexField(field);
default:
return field;
}
@@ -178,6 +180,24 @@ function fieldToStringField(field: Field): Field {
};
}
function fieldToComplexField(field: Field): Field {
const complexValues = field.values.toArray().slice();
for (let s = 0; s < complexValues.length; s++) {
try {
complexValues[s] = JSON.parse(complexValues[s]);
} catch {
complexValues[s] = null;
}
}
return {
...field,
type: FieldType.other,
values: new ArrayVector(complexValues),
};
}
/**
* Checks the first value. Assumes any number should be time fieldtype. Otherwise attempts to make the fieldtype time.
* @param field - field to ensure is a time fieldtype
public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx
+1
View File
@@ -29,6 +29,7 @@ export const ConvertFieldTypeTransformerEditor: React.FC<TransformerUIProps<Conv
{ value: FieldType.string, label: 'String' },
{ value: FieldType.time, label: 'Time' },
{ value: FieldType.boolean, label: 'Boolean' },
{ value: FieldType.other, label: 'JSON' },
];
const onSelectField = useCallback(