From cf6adb8b54a41c81455b6f8f5d5300a5c2b396ac Mon Sep 17 00:00:00 2001
From: Ryan McKinley <ryantxu@gmail.com>
Date: Wed, 11 Mar 2020 01:24:50 -0700
Subject: [PATCH] support duplicate field names in arrow format (#22705)

---
 .../src/dataframe/ArrowDataFrame.test.ts      |  22 +++++++++++++++-
 .../src/dataframe/ArrowDataFrame.ts           |  24 +++++++++++++++---
 .../__snapshots__/ArrowDataFrame.test.ts.snap |   2 +-
 .../__snapshots__/all_types.golden.arrow      | Bin 10354 -> 11114 bytes
 4 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
index d7beb342353..cf3da3d4fc4 100644
--- a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
+++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
@@ -71,7 +71,27 @@ describe('Read/Write arrow Table to DataFrame', () => {
     expect(after).toEqual(before);
   });
 
-  test('should parse output with dataframe', () => {
+  test('should support duplicate field names', () => {
+    const frame = toDataFrame({
+      name: 'Hello',
+      refId: 'XYZ',
+      fields: [
+        { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] },
+        { name: 'a', values: [1, 2, 3] },
+        { name: 'a', values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const table = grafanaDataFrameToArrowTable(frame);
+    expect(table.length).toEqual(frame.length);
+
+    // Now back to DataFrame
+    const before = JSON.stringify(toDataFrameDTO(frame), null, 2);
+    const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
+    expect(after).toEqual(before);
+  });
+
+  test('should read all types', () => {
     const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow');
     const arrow = fs.readFileSync(fullpath);
     const table = Table.from([arrow]);
diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
index 16c6ab3be36..311721a936b 100644
--- a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
+++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts
@@ -72,7 +72,7 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
       }
 
       fields.push({
-        name: col.name,
+        name: stripFieldNamePrefix(col.name),
         type,
         values,
         config: parseOptionalMeta(col.metadata.get('config')) || {},
@@ -91,6 +91,17 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
   };
 }
 
+// fieldNamePrefixSep is the delimiter used with fieldNamePrefix.
+const fieldNamePrefixSep = '🦥: ';
+
+function stripFieldNamePrefix(name: string): string {
+  const idx = name.indexOf(fieldNamePrefixSep);
+  if (idx > 0) {
+    return name.substring(idx + fieldNamePrefixSep.length);
+  }
+  return name;
+}
+
 function toArrowVector(field: Field): ArrowVector {
   // OR: Float64Vector.from([1, 2, 3]));
 
@@ -117,10 +128,17 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
   if (table instanceof Table) {
     return table as Table;
   }
+  // Make sure the names are unique
+  const names = new Set<string>();
 
   table = Table.new(
-    data.fields.map(field => {
-      const column = Column.new(field.name, toArrowVector(field));
+    data.fields.map((field, index) => {
+      let name = field.name;
+      if (names.has(field.name)) {
+        name = `${index}${fieldNamePrefixSep}${field.name}`;
+      }
+      names.add(name);
+      const column = Column.new(name, toArrowVector(field));
       if (field.labels) {
         column.metadata.set('labels', JSON.stringify(field.labels));
       }
diff --git a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
index ec232bff1b7..358c55e3907 100644
--- a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
+++ b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
@@ -103,7 +103,7 @@ Array [
 ]
 `;
 
-exports[`Read/Write arrow Table to DataFrame should parse output with dataframe 1`] = `
+exports[`Read/Write arrow Table to DataFrame should read all types 1`] = `
 Object {
   "fields": Array [
     Object {
diff --git a/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow b/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow
index d1732b1e78222e509fc26f21ecaf73b4dc7cf6fb..67f764eee6c3f3929578541172ad74ae2d91d495 100644
GIT binary patch
literal 11114
zcmeHNU2I%O6`r--G_(c+O_2qGUR_1Vm7Ln%y2f3V+9455ArVy@QA-rX*Y<UMle_m?
zyLW@r5Nx45R45W9icnOQwhsnX6+#GA$U+gSN`)ek2OfAreTe#iq6!i7S3)T)-#2q+
zeRtNo_5%+*>{w@K&OPVMoOAA(Gc$L5@X05?^7N=ub(D#ZDYZwj+@(IE`qka44=ttk
z>SwCZ3#=dQ-ST`l+E)R+fa8EcKp$Y{R=rLdc?*DpfI9${aN+TJ)T=^gzwEypBxg@#
zXP3gNkxznhh{x(x;X8^p`#9%U!=N&^_;gyCull90`HxkCg&+z1A^)8ptle7wE*jp(
z%Rl;|hej4wGym&DejJsf%rA`<$BO6C2lXq5S%6A?5-{^or9Ob7BY?HLVFTbYlo|kB
z_z-LbWH9EBcPjM^;O$*X9R*y6L5~2gqQfD;8l12faJdI#0G@~8eSnJ?gJVAX*Lt1y
zJ|O_h3v7c$(=TrrpX~?!DM0Zd8dCC-S{w(nad;w&%CIOXFI6?a^$}Rrw?3%X`4+a5
zd+E<cha-?7pxnjU>M#I0OjY{;9|si2j1IY>TgHB0#%{-7`I{M&@j*~(SCicrWz0cP
zIBxMl$L`rQjZd5j;u@pF_$qJvE0Eub+zWqcv3=KJ^k;3~3%a%a9_mpvdgO=ad(4l$
z4>Ij|PyOAsySG65?RXbtJlcI9#~XF*UWn5m8y~Z#yJJ`W!*|}ZBiZju>~S6uP<mNg
zUG{)Z+daT10L2lbi=#rzyMIq~=p<8lud^R6%GlNqqYrQ8hX;%8{O~OD7A>I<x08KI
zVv)9+SX90FcQOEa-VdV_IX`Tw(BcQgG;-oD{IL|Y58KIp`yZ}9dYa<mMbUx&_zYn5
zbB;f1QIZRb7F}jBAE$OG`xA-LTwL_z<6;nW>yLY>%ec|SQK3bLKZp*UWWIB|vme%E
z?1mrQd`KlaypvP^uGc;5hiznkE-~894<n#A`9bocqe6=x{v<kRKeUm#^v}+I__2)L
z@PnHlsl@2GogOVZFi&tz*RiyX>@OuY+xcMv^d>(@j&xLL(Sdn_I%q$%k$K_$&VG1Z
z#-<-W4H$LvC6$<HxS>S{<_YS)6WL!&Y|;;sCwlUEVhVKYhrQHA@};9fiw-7F^yxf7
z`A)L0{j1d<(~z+_`z_Id{^0uOrp@Lo(YkI<=yh8argkX*dx_H{;3uFo<%;8=(<bV|
zHBnM$lN#<Cb~UF%JDHz>hkl~}2^hAJ_rrA=*ZRRtmrx*=C0hM3g*l2&3m$%``*!m0
zNSwCw!!w}g{U9l`Ne$NzZ{>8LAGVVD1$a97;de5w^@E%8pg=B5wEE$QUdvU1^OSzr
zPX1kq&vt&82R-ixNq<dhxPG{v(}8~2O6E=Q&=1@@6BrMS3p+2~mT~vv8TZ3(%7X%p
zEa8rSRPQBJfqHKz`>w?0H^F-WfYS3Ee*hBz_Pq!m>jS+%Hp^#Tg{B;FCC87W7|uK8
zDDg`pLw-3}_DiEYzi^K+mnI9*Ni<M)<CL{q-3ib|>)viG?&=4Sb?!mkuVwr6KA-;F
z2gM0EsTBrL^Gx?5;4px;j3NFw#vnlFP-k0G%_>oHa_juNB(hVGeF$*Q4~_=2Vf^Lr
ztY4D$SGmK-v28X-F>OP!w&O4q##N;_XQ<CKUi>k@IX{cCI7EMPDD=#c&>MWB5}t|D
zTGgvArnPw9n+?5193wL?ET^ZU`^|xIE3RssW{biF%@(3K3oAT}`K5&*u7*SYd^i`C
z(ez72yd%CV9@pe`nI?a=1;^5PYDXNM#ZhvaN7zlZ52s0xh28=0_+scirH4M0Ce^TV
zCdi^R@xGYWl6l?a;Wnt846`r9L2}wJWtAFqsa0ZhznEo9)za{=)*W%ukbht<Ee|iI
z-#Cz^2NuJ4X?R|bH|(Fs897OflS}@_%U9NX>+7aC!`#^Zy_@*r3gE`q>v&h|0F*n}
zW_+yz9sw}EF3bAtq0M-dCDje9rulMBWcGuf^R<0ko!9fm#*UdYeR|F?hNvq;{~Qc=
z=T{#_q<)=r0$G1Qna1}3^0D;e6u_C*4{7#4O@mQ{-pg+(1EAP%DSW;R9(2(t<Be8p
zV?Z9+dP?wVsc9F>jQ+*<45^o;o|1Z9YFk;{%D*85i(hl_-&#SbExt$a8&X^On}S>V
zZNY72@d3-n#ExR+2L!kD5y1@?;zsa6dBFU@V&!KX`g4xF#VtR-<K*S9IQT1$yv413
zKXv%m1-JBTj=aUKeK#EWn}S=q4qT(6`PDhx<Y_(_6x`Y~;?SoBw_RA=_P^xFFAHwz
z&pYxKxBg#u<lhk7(r-HQ7PtCtIP$k0TqhE%JZqU}w*NuFZGR&Uog(}Wn4foiW^wy^
zEKVAW#k=%oe;Dk4?;rNd+|0Q9EMNlgDB!WwI~FBhdWyf^;6hsQ9{s#GK6d|zH|!-*
zjNfupog+P1&eJIuESq%$-ROePyXwON+<#31WH(_p6WsIEmaqzk-r2NP@oGslhh=?u
zAx$zbTJVz8(+lsA7gfFD$mqilsMS@qvVwlNKCxI_t>RDBSL!69Si1Whq*Vgr49ndL
z0nJT*q4QauP8>zcDRrZ6;}kq7_v!Ttjg`i@4!=Rox|Zk1PFo+Ca&j6ce?j+ZBy)5;
z;F#vQ{S|vi(3EC=!=;Hrw{o^_V^4jneyj{@(`Une?HAM6>LlE;LLi^*hqTSHHtCzs
z*yH4{D5KN{`e2znWa?WD|8C81^|`fT_$LoB{#fkzxPE9Zmv$Ze-f!N#S6%2oeF?RZ
zYuM8muOVdl<$=r3b*LY^5}pXS7vWxo&xQb;V@#;j{Rk%n%8VpFyPq*n`P;o4>fQ}C
z-;Z?fhPJ*(>fQ~R+q~}G5U%)gUnjeFL*|mVdpBhK(7hWne(2r}<vv<tr`5e1GXD<J
oy&E!q=-v$(KXmVgx_3j}yP@vg5WhzJzuXPY;Je8|{Xe$<1sk#9CIA2c

literal 10354
zcmeHNUu+yl8K1Kq*Q7@YAq`fE#A;PluH?jyF;+N1?KDvv+D24uMXdsg*Y;k#WN+8H
z+Y1*tk^R6!RaGKXJVYp{6)K2_JXBR5EW|?veTc+Mg-{==K3Iqc3n5}4KnR87_uHNC
zZf^GO^1=(!jrHBkx8J|{X1;HBX2&O=d+uv59#^V~vi&Kg4k(uW>H$?$_p1@Klsc3?
zQ-xt*MYPBC^D(r40T>3n0GI%b0DimIY?4Oa0^lTIAE4@&zq*S0wx&}r^;f*;(z)c)
zx?juWBd_A)u{!eSOwTiDQ^$r|^S$cI+KX{@wdR(-;XYaQ%3kETlkV?+^yaPY@1f!B
zz5dgmIA|n(Epfj!>4rfiNZiuw%<Rk+=wSaUe&V504*_m}QmO0rE471;d=s$y38hv6
zdvG8G+__Jw1n}{wQkMZmz^icP2;e%DO#*IU0A>O23@LRQ@E)9A0{kBOF9T@v!`sa!
z{XL@ro)_2#TS_1MOXlkl;GY3R^)U37L;qY7RN#qMS+AvhGB#jQfBLxD<mmU}zYCqT
zxu5~nezwKtWo=XD!7!Dx_0qpHTbwVG@9n>3HaJHFrA9ma+R-)~P-v&+*K!<(=Pr6-
zowMFK@5KM8``(NHgTLl{8~rlm#J5GzF@32`Os_J{aX!p(B(Kf$%HM6jh70sh^lxkZ
z4xW9M`eeS!VeBPyv*Ma<>&O4$I1crdd0WTIY3L=O46_#7v!K(raliur%Vciu4kNc|
zvm0;qAN~D&L)&s-C-3JMW_tPgN@}yvSLTO0cIaCcJ8IY*<0GKw{A`=d`H9%W+BAwk
zmO|TKLuUDo?e9>>yk6Bd9cVZVXw7Rqh>U5?+5Qc3ap4{^5>v*!4w>^>M9hi5SeBG!
zGH3G%<Roo&<9+I%{rzlcTMp!Nej4L70=k(iKZnhG$w*A~^7D2VKMj*PKaU}2XjA+&
zd1vnS_j3vH%`xrT=i_q{zB!xBQ@#sxV`Ip?q+_X<pC4)4%v0&RA)d3zJf+QUybt}W
zzn^E&7xD8!=;(;s6D>D$HkqfC>nHQLjwSl3^K{6})AzJ(`dI`pZVi(;n^~TYq<Ko)
z-DLJ7pZLC@9Rk*`!n635HSh%b)Yae4G|kz5IellU!UA-4_-x6|>9{iI_KabFTiX|(
zE%BUPmA*eSyY$!Qzk3gVC(xf>{+jtP0^Mf5Hnd#_&uxEAxkq$-_44;;+P>khDW3Cp
z&#<fg?co0d=>6t*9R2C#Z@(N`OW&`mz&x`3HRTTL@$TjCHErMU*A&nByJOhZ{&w(x
z2zoz%$Iu_~w_~4#K<SrDi|Kn-6)4+JZd8wV0zU3ZQHHko1DFOdZeG=WpdRjrOw(pu
zy`%ZKSN$|#!>t67TbiD9D_+Ab9p^rq@8Fd<DhKD$Kp8`pkE#0rX-WHQ+feEt_B4mE
zpFNna2|Q;U1Uvh42puKhBvCkknrDt50J!I4-_F248NZG-yOvag=zMDdH2t`&Wfma+
zMZkvZJ>xC=;nV&lx1`%&=MELeiDiS5wKIzNip{zoVki5)*13p<>MUTxO@bu!p-&%H
zob0ILOgvllF9vbF=G4~WdbsK=`_3AUH;Lm{;%^6!W(S#WT(vMx)-)G-Ru00%ukx(m
zmdajO^C#U^e<i4(>6T_NC!8C;4)pY7P5!1HoQ+r6JK|6l2GIo`t=io_9Y<c`J4c-t
z)_mvr)by!1s`=H6UJ}HS^F&;aR?{XAU0(IPpFAFV(FM1ZRO{$Vy&6LOT9T~SN>fv5
z-w_ur`C}__Woj+{&e0@3y5@)LQ>&@HDfbG_Zc&^Ka>?C${o0$ZjI|k8hY<h$=38Ql
zrz+;98B;fazXd?q$2McC0hk3aPOj^BBi8`FldVund1S4<sd*m;-{%0mW9dqo8xk*B
zUW}x9!T4n#8Ri>s*3O|3SY*Gt^#$gei#U@qS6GVJxiFK8kf6>*I)|d^!#vB6gCb{n
zK5u{rDq3Z()yg+GCM;4fq!{ezTIac1(zx(7G`^tgZCy(xxX8b&34-6W@b^Vf*Mc9=
z_>Qhcepll{zoT)f1RobZZa7dxeq7^1pVoNBg|$@U6Z!$y0Tz*8vgj{c@`4NhRZISw
zg}-6R3oib>W$|xoT<A9~dBMfM9ZP;!<3dk^HFGihP1oq`TM!>iXk7f6w&;r*7Zrkw
z{&h>fp>d&Kwd4ht@!z)O-_^L#cP)9rrN4WY{2dEV7Y*q=Yn^AJe?sG;Z`z_$gx{ge
zyyG*$WqJfBjYaSQy>mP=?0-K#)XVjlarX$|1mH2ild*F)h@N_$zs^KCt~!r>#hIIZ
zblRD6q9DX?C8({C?lnw0<-A7ru7H}f;4@>owsU>wzG@zzcMaL5f!{Xk>v-Gx&ZW3s
zb?Q;Dg4g#{IgS!1C_7Q?q%YV>C#X3y)5pJbOl@tc%}waxJBdYbwS_;`+-#DFB6RyX
zNLvKP8D24)1T@?Hn$Bk?oj6J=hp($miBs^Pd_Mi&jK*fmuEn3B*>`D^A16S5%rfn#
zIQa{y)=aWfFKuSe<yZSdLQ3i6&$v>eri+}^J^X2Q(<3shGo3B}({agkZOvmiHVNbt
zJ*2g5wdw6=)Hweu%HXRtbg;~y%IaIK@$Jqp{kgT7@z0-P?6HV_d<yR@uZ^Dk{;z-e
zp!#m{!fU8Axt2dIdo3a1*AHxdzK4p~m2k(;y$JUzd^QQ--ku98`+gI56D&)5;gkJL
z_Edfc_Y;HriNXEE2-0D2Kf!yL_9rKfiJaBAS`6+daJOh(FbwV|vUC67ej>Y+wR2{0
zKXI{jUp=^==)8{^+)wEL*T6?}^TWTLLxcN?_BfgM8iV_Z>_1`5cp2PJU`z)06Wp%u
YyMZ5mI>aZV|I7P{U7X)frvGdBUmMP6*#H0l