diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts index 9137198c95c..975ddf9bd96 100644 --- a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts +++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts @@ -1,6 +1,10 @@ +import fs from 'fs'; +import path from 'path'; + import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame'; import { toDataFrameDTO, toDataFrame } from './processDataFrame'; import { FieldType } from '../types'; +import { Table } from 'apache-arrow'; /* tslint:disable */ const resp = { @@ -59,4 +63,12 @@ describe('Read/Write arrow Table to DataFrame', () => { const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2); expect(after).toEqual(before); }); + + test('should parse output with dataframe', () => { + const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow'); + const arrow = fs.readFileSync(fullpath); + const table = Table.from([arrow]); + const frame = arrowTableToDataFrame(table); + expect(toDataFrameDTO(frame)).toMatchSnapshot(); + }); }); diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts index 2963f3d9136..aa78b1f20a4 100644 --- a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts +++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts @@ -1,4 +1,4 @@ -import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types'; +import { DataFrame, FieldType, Field, Vector } from '../types'; import { Table, ArrowType, @@ -28,6 +28,17 @@ function valueOrUndefined(val?: string) { return val ? val : undefined; } +function parseOptionalMeta(str?: string): any { + if (str && str.length && str !== '{}') { + try { + return JSON.parse(str); + } catch (err) { + console.warn('Error reading JSON from arrow metadata: ', str); + } + } + return undefined; +} + export function arrowTableToDataFrame(table: Table): ArrowDataFrame { const fields: Field[] = []; @@ -59,35 +70,23 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { default: console.log('UNKNOWN Type:', schema); } - const labelsJson = col.metadata.get('labels'); - const configJson = col.metadata.get('config'); - - let config: FieldConfig = {}; - let labels: Labels | undefined = undefined; - if (labelsJson) { - labels = JSON.parse(labelsJson); - } - if (configJson) { - config = JSON.parse(configJson); - } fields.push({ name: col.name, type, - config, values, - labels, + config: parseOptionalMeta(col.metadata.get('config')) || {}, + labels: parseOptionalMeta(col.metadata.get('labels')), }); } } const meta = table.schema.metadata; - const metaJson = valueOrUndefined(meta.get('meta')); return { fields, length: table.length, refId: valueOrUndefined(meta.get('refId')), name: valueOrUndefined(meta.get('name')), - meta: metaJson ? JSON.parse(metaJson) : undefined, + meta: parseOptionalMeta(meta.get('meta')), table, }; } diff --git a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap index ae779698246..e3b27e0775e 100644 --- a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap +++ b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap @@ -102,3 +102,325 @@ Array [ }, ] `; + +exports[`Read/Write arrow Table to DataFrame should parse output with dataframe 1`] = ` +Object { + "fields": Array [ + Object { + "config": Object {}, + "labels": undefined, + "name": "string_values", + "type": "string", + "values": Array [ + "Grafana", + "❤️", + "Transforms", + ], + }, + Object { + "config": Object { + "decimals": 2, + "filterable": false, + "links": Array [ + Object { + "targetBlank": true, + "title": "Donate - The Sloth Conservation Foundation", + "url": "https://slothconservation.com/how-to-help/donate/", + }, + ], + "max": null, + "min": null, + "noValue": "😤", + "nullValueMode": "null", + "title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)", + }, + "labels": undefined, + "name": "nullable_string_values", + "type": "string", + "values": Array [ + "🦥", + null, + "update your unicode/font if no sloth, is 2019.", + ], + }, + Object { + "config": Object { + "max": 1, + "min": 0, + }, + "labels": undefined, + "name": "int8_values", + "type": "number", + "values": Array [ + -128, + 1, + 127, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_int8_values", + "type": "number", + "values": Array [ + -128, + null, + 127, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "int16_values", + "type": "number", + "values": Array [ + -32768, + 1, + 32767, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_int16_values", + "type": "number", + "values": Array [ + -32768, + null, + 32767, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "int32_values", + "type": "number", + "values": Array [ + -2147483648, + 1, + 2147483647, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_int32_values", + "type": "number", + "values": Array [ + -2147483648, + null, + 2147483647, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "int64_values", + "type": "number", + "values": Array [ + "\\"-9223372036854775808\\"", + "\\"1\\"", + "\\"9223372036854775807\\"", + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_int64_values", + "type": "number", + "values": Array [ + "\\"-9223372036854775808\\"", + null, + "\\"9223372036854775807\\"", + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "uint8_values", + "type": "number", + "values": Array [ + 0, + 1, + 255, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_uint8_values", + "type": "number", + "values": Array [ + 0, + null, + 255, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "uint16_values", + "type": "number", + "values": Array [ + 0, + 1, + 65535, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_uint16_values", + "type": "number", + "values": Array [ + 0, + null, + 65535, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "uint32_values", + "type": "number", + "values": Array [ + 0, + 1, + 4294967295, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_uint32_values", + "type": "number", + "values": Array [ + 0, + null, + 4294967295, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "uint64_values", + "type": "number", + "values": Array [ + "\\"0\\"", + "\\"1\\"", + "\\"18446744073709551615\\"", + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_uint64_values", + "type": "number", + "values": Array [ + "\\"0\\"", + null, + "\\"18446744073709551615\\"", + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "float32_values", + "type": "number", + "values": Array [ + 1.401298464324817e-45, + 1, + 3.4028234663852886e+38, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_float32_values", + "type": "number", + "values": Array [ + 1.401298464324817e-45, + null, + 3.4028234663852886e+38, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "float64_values", + "type": "number", + "values": Array [ + 5e-324, + 1, + 1.7976931348623157e+308, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_float64_values", + "type": "number", + "values": Array [ + 5e-324, + null, + 1.7976931348623157e+308, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "bool_values", + "type": "boolean", + "values": Array [ + true, + true, + false, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_bool_values", + "type": "boolean", + "values": Array [ + true, + null, + false, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "timestamps", + "type": "time", + "values": Array [ + 1568039445000, + 1568039450000, + 1568039455000, + ], + }, + Object { + "config": Object {}, + "labels": undefined, + "name": "nullable_timestamps", + "type": "time", + "values": Array [ + 1568039445000, + null, + 1568039455000, + ], + }, + ], + "meta": Object { + "limit": 4242, + "searchWords": Array [ + "Grafana", + "❤️", + " 🦥 ", + "test", + ], + }, + "name": "many_types", + "refId": "A", +} +`; diff --git a/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow b/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow new file mode 100644 index 00000000000..ecdb42c8f72 Binary files /dev/null and b/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow differ