From 29e9b1f7744423df80538f9d96294dad62f4d7ae Mon Sep 17 00:00:00 2001
From: Ryan McKinley <ryantxu@gmail.com>
Date: Wed, 15 Jan 2020 14:33:56 -0800
Subject: [PATCH] DataFrames: add arrow test and capture metadata parsing
 errors (#21524)

---
 .../src/dataframe/ArrowDataFrame.test.ts      |  12 +
 .../src/dataframe/ArrowDataFrame.ts           |  31 +-
 .../__snapshots__/ArrowDataFrame.test.ts.snap | 322 ++++++++++++++++++
 .../__snapshots__/all_types.golden.arrow      | Bin 0 -> 11778 bytes
 4 files changed, 349 insertions(+), 16 deletions(-)
 create mode 100644 packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow

diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
index 9137198c95c..975ddf9bd96 100644
--- a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
+++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
@@ -1,6 +1,10 @@
+import fs from 'fs';
+import path from 'path';
+
 import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
 import { toDataFrameDTO, toDataFrame } from './processDataFrame';
 import { FieldType } from '../types';
+import { Table } from 'apache-arrow';
 
 /* tslint:disable */
 const resp = {
@@ -59,4 +63,12 @@ describe('Read/Write arrow Table to DataFrame', () => {
     const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
     expect(after).toEqual(before);
   });
+
+  test('should parse output with dataframe', () => {
+    const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow');
+    const arrow = fs.readFileSync(fullpath);
+    const table = Table.from([arrow]);
+    const frame = arrowTableToDataFrame(table);
+    expect(toDataFrameDTO(frame)).toMatchSnapshot();
+  });
 });
diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
index 2963f3d9136..aa78b1f20a4 100644
--- a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
+++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
@@ -1,4 +1,4 @@
-import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
+import { DataFrame, FieldType, Field, Vector } from '../types';
 import {
   Table,
   ArrowType,
@@ -28,6 +28,17 @@ function valueOrUndefined(val?: string) {
   return val ? val : undefined;
 }
 
+function parseOptionalMeta(str?: string): any {
+  if (str && str.length && str !== '{}') {
+    try {
+      return JSON.parse(str);
+    } catch (err) {
+      console.warn('Error reading JSON from arrow metadata: ', str);
+    }
+  }
+  return undefined;
+}
+
 export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
   const fields: Field[] = [];
 
@@ -59,35 +70,23 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
         default:
           console.log('UNKNOWN Type:', schema);
       }
-      const labelsJson = col.metadata.get('labels');
-      const configJson = col.metadata.get('config');
-
-      let config: FieldConfig = {};
-      let labels: Labels | undefined = undefined;
-      if (labelsJson) {
-        labels = JSON.parse(labelsJson);
-      }
-      if (configJson) {
-        config = JSON.parse(configJson);
-      }
 
       fields.push({
         name: col.name,
         type,
-        config,
         values,
-        labels,
+        config: parseOptionalMeta(col.metadata.get('config')) || {},
+        labels: parseOptionalMeta(col.metadata.get('labels')),
       });
     }
   }
   const meta = table.schema.metadata;
-  const metaJson = valueOrUndefined(meta.get('meta'));
   return {
     fields,
     length: table.length,
     refId: valueOrUndefined(meta.get('refId')),
     name: valueOrUndefined(meta.get('name')),
-    meta: metaJson ? JSON.parse(metaJson) : undefined,
+    meta: parseOptionalMeta(meta.get('meta')),
     table,
   };
 }
diff --git a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
index ae779698246..e3b27e0775e 100644
--- a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
+++ b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
@@ -102,3 +102,325 @@ Array [
   },
 ]
 `;
+
+exports[`Read/Write arrow Table to DataFrame should parse output with dataframe 1`] = `
+Object {
+  "fields": Array [
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "string_values",
+      "type": "string",
+      "values": Array [
+        "Grafana",
+        "❤️",
+        "Transforms",
+      ],
+    },
+    Object {
+      "config": Object {
+        "decimals": 2,
+        "filterable": false,
+        "links": Array [
+          Object {
+            "targetBlank": true,
+            "title": "Donate - The Sloth Conservation Foundation",
+            "url": "https://slothconservation.com/how-to-help/donate/",
+          },
+        ],
+        "max": null,
+        "min": null,
+        "noValue": "😤",
+        "nullValueMode": "null",
+        "title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
+      },
+      "labels": undefined,
+      "name": "nullable_string_values",
+      "type": "string",
+      "values": Array [
+        "🦥",
+        null,
+        "update your unicode/font if no sloth, is 2019.",
+      ],
+    },
+    Object {
+      "config": Object {
+        "max": 1,
+        "min": 0,
+      },
+      "labels": undefined,
+      "name": "int8_values",
+      "type": "number",
+      "values": Array [
+        -128,
+        1,
+        127,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int8_values",
+      "type": "number",
+      "values": Array [
+        -128,
+        null,
+        127,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int16_values",
+      "type": "number",
+      "values": Array [
+        -32768,
+        1,
+        32767,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int16_values",
+      "type": "number",
+      "values": Array [
+        -32768,
+        null,
+        32767,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int32_values",
+      "type": "number",
+      "values": Array [
+        -2147483648,
+        1,
+        2147483647,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int32_values",
+      "type": "number",
+      "values": Array [
+        -2147483648,
+        null,
+        2147483647,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int64_values",
+      "type": "number",
+      "values": Array [
+        "\\"-9223372036854775808\\"",
+        "\\"1\\"",
+        "\\"9223372036854775807\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int64_values",
+      "type": "number",
+      "values": Array [
+        "\\"-9223372036854775808\\"",
+        null,
+        "\\"9223372036854775807\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint8_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        255,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint8_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        255,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint16_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        65535,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint16_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        65535,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint32_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        4294967295,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint32_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        4294967295,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint64_values",
+      "type": "number",
+      "values": Array [
+        "\\"0\\"",
+        "\\"1\\"",
+        "\\"18446744073709551615\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint64_values",
+      "type": "number",
+      "values": Array [
+        "\\"0\\"",
+        null,
+        "\\"18446744073709551615\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "float32_values",
+      "type": "number",
+      "values": Array [
+        1.401298464324817e-45,
+        1,
+        3.4028234663852886e+38,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_float32_values",
+      "type": "number",
+      "values": Array [
+        1.401298464324817e-45,
+        null,
+        3.4028234663852886e+38,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "float64_values",
+      "type": "number",
+      "values": Array [
+        5e-324,
+        1,
+        1.7976931348623157e+308,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_float64_values",
+      "type": "number",
+      "values": Array [
+        5e-324,
+        null,
+        1.7976931348623157e+308,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "bool_values",
+      "type": "boolean",
+      "values": Array [
+        true,
+        true,
+        false,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_bool_values",
+      "type": "boolean",
+      "values": Array [
+        true,
+        null,
+        false,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "timestamps",
+      "type": "time",
+      "values": Array [
+        1568039445000,
+        1568039450000,
+        1568039455000,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_timestamps",
+      "type": "time",
+      "values": Array [
+        1568039445000,
+        null,
+        1568039455000,
+      ],
+    },
+  ],
+  "meta": Object {
+    "limit": 4242,
+    "searchWords": Array [
+      "Grafana",
+      "❤️",
+      " 🦥 ",
+      "test",
+    ],
+  },
+  "name": "many_types",
+  "refId": "A",
+}
+`;
diff --git a/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow b/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..ecdb42c8f7226bb12f89472046b5b0f8bef42a77
GIT binary patch
literal 11778
zcmeHNUu+yl8K1L_YuY0+Aq5NdVYTAHNSFFztjL@KBn3ecsfa?<qGA+p?7iHPy<O{W
z&p3^$SQQU_h(riMvQVLZ2$dgrKvf?i)Q6~wP^DU^52)$`4^^M4NF}BOFc3I?zuo!n
zX2+L3Kk$NdBYn5?&3yCCZ+`P<cV_Oy>C@kS=7>_oPboF27#>y!R8c*m#t>8LP?Dw!
zqtJ?oPw4as#9xPuLe4;@A!CsB4?7*^(Y63N0r?oD=~utGjOV7#r(gPSc;Usf@x>Lt
zm6i{^hL2?RrnWnYIDK4lTfWy^sy!1mms@V-d+s-yUeyadx8%P6iyL=u{umKw=jB&^
z<scIKt=Rp3$qnj_dhAx_X6I&~hY#Xw__2qF`Yhz+CzZN?K&i(d@h6n}BS6nVUV22S
zdjPipdF|s$O+jKb>b-}QItJMqhcCzq8nz9mWylo}u{#1TAlqno39^YgSm$$p?sVAR
zKnq%4U>v4u9zvU&e@c0K3~_XY3fn>8oeTW4alL`oc#V}-Vki83pzQ&-8vse6kRO0c
zhL8T$>9B_$<nMQPQ(Viir64@aCvlY_#B~Zb>`&=015O6dnzqZs^TPcU5BuHp^Yf2V
z94kntC^g>Y=NL5Ncvt&(5GR%{Kj)$-IJ@cvZH}PSRyZo|^u{9d7;KKlV3X(Xj?O19
z6om0b{FofYx*qz$&`~BNVi!Cw?E}ws*w{XP`xl#^(E@oQ2H|H{`*x7_8Az9(Y7lwx
z(K(^W#8bk!l(~u`NwE1j`XBb$NUqNVy8mkMKtUMgllbN!taC%xHHA2)o3dbK#?tM5
z;QGDB)z8<RdxQC!24=S1f$Zh$3$y+BTF_<k#(1?uz6!ZMzpVR{9Hrx3jY@wCUvKKV
zGJN%5W%#<c4_qAi6A<<<@pXAKe7b+Lc^la?&K7}Pc>6Rwb;nt|9vZ_egYUX7lgHaT
zSgVf1U!KoZ-LK@0f-u6TInESph>yHsNmmxE46c&4&BGO9%{u-7U#I^*n6In44hKH=
zw9Vk&oR_!vfouHVoz5ViulzN|m&aEbHa1@dtPEcZ+BT1`Uus;!z3J<P-NAg_(slH0
z+mWL(Ei<_8?E@FrvB`dYVEBATkN-Tr7GY!aWw^-Tx}t6K_~JT7T*8;>>(ze@=4(gS
zLB1Y^Wai_1m#z%1&tcrM4SBe@j?pI1=eX`q@}<|Y5qllGs_Wu>WPO~E2CNLO*Y|-d
zUB||fbxhh|`@Hkdy}ZS!+k9V613Sw*kW3r1%UA~Af-aNC+m+;-OBId}@cV@BTXIK1
z7zy|Tk$1qhu1~nLpk=V_?gQI&^8J-!V;gLLF7gi{oqd@0=SackZ4tO1#G5(S&GqcM
zE|bUGZH>)A-FAPNexK8Qo5$N2Y>3g|&2*9Bt*mYHuq|RugS&!*AI)3F-?PXc#M_3h
zhd%7~nRDLcZF?Ws?rUt51N^><aY>AMyd8s$#hVE&!&^<;=JB?Xe50uXv1#5i{;nZ^
z5N~hmdK~zmZFhakSx3e&me`IwY;(zXnkvvP&+i>QCpKWW2|*aSfgjoTmJw&4t?4@H
zi{I}iJhM+X2VlFU^SKYi_7wZxMm=;Z<&xX**4)Yw?qm54w-kld`gueUCeYFe^$;Wp
zNt<j8N*%;}IE206!Gss?;}61|c#BwcDA=hNCh%n2u0UjOaRy_Iz8y&B8>SUE>*4wC
zOU7*Hi@MA_%9J49Np4KP<zM_#8he^MwvuzMo1DgyWSA#yl)da9w6E`CPyH0+k{j3K
zz;`Qh2;ihg0M7KOroUQ`+AXJ5i`v1mbIx~aI5vnKzY+bQ{`vHPpr2JMh~k>I!U00H
z9>ji=`){{W^@5gPa+m$3dIM3nGK;=opKvJZZb_s3O&ai2v`lo=;Vh_!7kFgRgZ;HA
z^kUyR?3}6j&UcckzZ!)tzq#tg^(b_{9<{^eB+A1KuX)~&p9sA0f?J83V9jke1Guln
z@k*;QGm~J)c<7dYd?{+o)S@3Aj-$ghKUkSrPU@R+pU0U<7$p!<e&^+l8*abxG=;g5
z*UvJRcv8VRW{#yT=)YzMVlXkzu@pn*ARHf?`n!T_GrxJPA(UBTwzaJTJFc01#}LxZ
zJ!?7_#*(?fF-v?L+LzE!d)|zpHe%|BV*zJ7oEr>9jC#(TBCP4}$z-l{^GA6$HiaBH
z8)LaOG@Naha8x3Fo?IUp<VkKArqf5#7Dl<&Fo@mpL`%<*7+ur4JVYlC4C~1+`zsAE
zTKZM3%R_Xr->~dAEqzN0>Zj;}_oiikOY4&Vw$|k#`jk%iKtBb~j@BiA0&9aAL>K!>
zoiF;7r8`!6(Zzn+vM*cuyj5OwY2TBU{VA<W{-RZ0bZOt3ReoLTlD}b<7hU{sS>=DD
zb;-YFl^0$7ziX9$PwSGuYn2yW_)8`tI!JWBYZyeI)OwnVajtdoKW*8Ut$dc_xvn|y
zm?pZw6P<Ysq7U=4e5Bg{o*(gxqx59$<@uXF&QD+DXPREvsz%L5OJN!A2CmhVSqP8%
z+beiU`p(6u-E`VveF<;fnQ9cqPQB`ck(0a^OHRGz%$ARQ@$uBndf{>s5x?sy)fFaF
z2T@7qVo4nVIf!>st(&ixQ%O%OwJqaFXSi(GGkKjq3~4re7CMREvrBcJ(Q((&OJ|wH
z6gg>mYGyul`sGuCjA>n8$F>(c$xrRqzXIr&nJ4x=KWVz?Y55)4Gu-XAH=8f*y?a^a
zn(bFz8$a=rx88VE{it~18lS5NfA&*6#jcw#fOS6Ct|H#f{Ho>qm+xn$l_2vFj$UG!
z;w1>eqIM|n@HCab!~3n_{Z{r~iffJB-3;%y((hgSZp!3nc)yjNId$(=de?4PnR;Vs
zc)x}F{o(zVf7ZI38s2a9{<|=|-|DRw!}~3z9IT(2b93v`$6Q~B_ggqS()U4oc(X2j
zOx}j~Ti72A@3(k&F}&Xz-ft!U>nvm4<r*=(-%5Fq{owF^Yk0r4g?YjAtpCURt$Cb>
Jok;%s{a+nu6=nbc

literal 0
HcmV?d00001