public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13 from grafana/replace-directives

Browse Source 
Compilation fixes
This commit is contained in:
Owen Diehl
2021-02-23 22:03:59 -05:00
committed by GitHub
parent e9ec87dc67 f889b7b3ef
commit 27b287f587
3125 changed files with 66 additions and 860078 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
vendor/
go.mod
-4
View File
@@ -10,7 +10,6 @@ require (
github.com/prometheus/alertmanager v0.21.1-0.20210211203738-a7ca7b1d2951
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/common v0.15.0
github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
@@ -32,9 +31,6 @@ replace github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0
// Use fork of gocql that has gokit logs and Prometheus metrics.
replace github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85
// We can't upgrade to grpc 1.30.0 until go.etcd.io/etcd will support it.
replace google.golang.org/grpc => google.golang.org/grpc v1.29.1
// We can't upgrade until grpc upgrade is unblocked.
replace github.com/sercand/kuberesolver => github.com/sercand/kuberesolver v2.4.0+incompatible
go.sum
+53 -1
View File
@@ -1,3 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -238,7 +239,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
@@ -344,7 +348,12 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
@@ -414,7 +423,9 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-macaron/binding v0.0.0-20190806013118-0b4f37bab25b h1:U65wj9SF7qUBTGrnt6VxbHCT0Dw8dz4uch52G+5SdfA=
github.com/go-macaron/binding v0.0.0-20190806013118-0b4f37bab25b/go.mod h1:AG8Z6qkQM8s47aUDJOco/SNwJ8Czif2hMm7rc0abDog=
github.com/go-macaron/gzip v0.0.0-20160222043647-cad1c6580a07 h1:YSIA98PevNf1NtCa/J6cz7gjzpz99WVAOa9Eg0klKps=
github.com/go-macaron/gzip v0.0.0-20160222043647-cad1c6580a07/go.mod h1://cJFfDp/70L0oTNAMB+M8Jd0rpuIx/55iARuJ6StwE=
github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191 h1:NjHlg70DuOkcAMqgt0+XA+NHwtu66MkTVVgR4fFWbcI=
github.com/go-macaron/inject v0.0.0-20160627170012-d8a0b8677191/go.mod h1:VFI2o2q9kYsC4o7VP1HrEVosiZZTd+MVT3YZx4gqvJw=
@@ -595,6 +606,7 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
@@ -633,7 +645,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs=
github.com/gosimple/slug v1.9.0/go.mod h1:AMZ+sOVe65uByN3kgEyf9WEBKBCSS+dJjMX9x4vDJbg=
github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85/go.mod h1:crI9WX6p0IhrqB+DqIUHulRW853PaNFf7o4UprV//3I=
github.com/grafana/grafana v1.9.1 h1:4ykSOyQ+qImmZmILEgqsokVnWmvR8/A0zt7jN2RAQxM=
github.com/grafana/grafana v1.9.2-0.20210217182004-6c4be29655a6 h1:qb6jgR/WKFRaISkkAbXU5YaZcp5oeElRspSzGNZBJZU=
github.com/grafana/grafana v1.9.2-0.20210217182004-6c4be29655a6/go.mod h1:krMdqLnTBCyxzi+jRQ2LVlWngfWg2pihNlPD+VzzBww=
github.com/grafana/grafana-aws-sdk v0.1.0 h1:25TSkS57lvVen3nTCNMu5SdI8ju5Z0f7cvCsukhlqJc=
@@ -839,6 +850,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA=
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
@@ -885,6 +897,7 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY=
github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
@@ -1326,6 +1339,7 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw=
github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY=
github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099/go.mod h1:hz10LOsAdzC3K/iXaKoFxOKTDRgxJl+BTGX1GY+TzO4=
github.com/weaveworks/common v0.0.0-20201119133501-0619918236ec h1:5JmevdpzK10Z2ua0VDToj7Kg2+/t0FzdYBjsurYRE8k=
github.com/weaveworks/common v0.0.0-20201119133501-0619918236ec/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs=
github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
@@ -1452,11 +1466,13 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp v0.0.0-20200821190819-94841d0725da h1:vfV2BR+q1+/jmgJR30Ms3RHbryruQ3Yd83lLAAue9cs=
golang.org/x/exp v0.0.0-20200821190819-94841d0725da/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -1568,6 +1584,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVs
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1675,13 +1692,16 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190118193359-16909d206f00/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1774,6 +1794,7 @@ gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
@@ -1801,6 +1822,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q
google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM=
google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1862,8 +1884,35 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210203152818-3206188e46ba h1:np3A9jnmE/eMtrOwwvUycmQ1XoLyj5nqZ41bAyYLqJ0=
google.golang.org/genproto v0.0.0-20210203152818-3206188e46ba/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1878,6 +1927,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1904,6 +1954,7 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ldap.v3 v3.0.2 h1:R6RBtabK6e1GO0eQKtkyOFbAHO73QesLzI2w2DZ6b9w=
gopkg.in/ldap.v3 v3.0.2/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
gopkg.in/macaron.v1 v1.3.4/go.mod h1:/RoHTdC8ALpyJ3+QR36mKjwnT1F1dyYtsGM9Ate6ZFI=
gopkg.in/macaron.v1 v1.4.0 h1:RJHC09fAnQ8tuGUiZNjG0uyL1BWSdSWd9SpufIcEArQ=
@@ -1939,6 +1990,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
pkg/api/cortex-ruler.go
+12 -12
View File
@@ -1,9 +1,9 @@
package api
import (
"github.com/grafana/grafana/pkg/services/ngalert"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/rulefmt"
)
// swagger:route Get /api/v1/rules ruler RouteGetRulesConfig
@@ -99,9 +99,9 @@ type RuleGroupConfig struct {
}
type ApiRuleNode struct {
Record string `yaml:"record,omitempty" json:"record,omitempty"`
Alert string `yaml:"alert,omitempty" json:"alert,omitempty"`
Expr string `yaml:"expr" json:"expr"`
Record string `yaml:"record,omitempty" json:"record,omitempty"`
Alert string `yaml:"alert,omitempty" json:"alert,omitempty"`
Expr string `yaml:"expr" json:"expr"`
For model.Duration `yaml:"for,omitempty" json:"for,omitempty"`
Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
Annotations map[string]string `yaml:"annotations,omitempty" json:"annotations,omitempty"`
@@ -114,17 +114,17 @@ type ExtendedRuleNode struct {
}
// UpsertAlertDefinitionCommand is copy of the unexported struct:
https://github.com/grafana/grafana/blob/debb82e12417e82a0e2bd09e1a450065f884c1bc/pkg/services/ngalert/models.go#L85
// https://github.com/grafana/grafana/blob/debb82e12417e82a0e2bd09e1a450065f884c1bc/pkg/services/ngalert/models.go#L85
type UpsertAlertDefinitionCommand struct {
Title string `json:"title" yaml:"title"`
Title string `json:"title" yaml:"title"`
// OrgID is an obsolete field (it will derive from the x-grafana-org-id header)
OrgID int64 `json:"-" yaml:"-"`
OrgID int64 `json:"-" yaml:"-"`
// Condition is the refID of the query or expression to be evaluated
Condition string `json:"condition" yaml:"condition"`
Condition string `json:"condition" yaml:"condition"`
// Data is an array of the queries and expressions
Data []eval.AlertQuery `json:"data" yaml:"data"`
Data []eval.AlertQuery `json:"data" yaml:"data"`
// IntervalSeconds is an obsolete field (it will derive from the ruleGroup interval)
IntervalSeconds *int64 `json:"-" yaml:"-"`
IntervalSeconds *int64 `json:"-" yaml:"-"`
// UID is set only for existing definitions
UID string `json:"uid" yaml:"uid"`
@@ -158,7 +158,7 @@ type ExtendedUpsertAlertDefinitionCommand struct {
ExecutionErrorState ExecutionErrorState `json:"exec_err_state" yaml:"exec_err_state"`
Settings map[string]interface{} `json:"settings" yaml:"settings"`
// internal state
FolderUID string `json:"-" yaml:"-"`
FolderUID string `json:"-" yaml:"-"`
DatasourceUIDs []string `json:"-" yaml:"-"`
RuleGroupUID string `json:"-" yaml:"-"`
RuleGroupUID string `json:"-" yaml:"-"`
}
vendor/github.com/FZambia/eagle/.gitignore
Generated
-12
View File
@@ -1,12 +0,0 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
vendor/github.com/FZambia/eagle/LICENSE
Generated Vendored
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2018 Alexander Emelin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/github.com/FZambia/eagle/README.md
Generated Vendored
-7
View File
@@ -1,7 +0,0 @@
# eagle
Package eagle provides a functionality to export Prometheus metrics aggregated over configured time interval. This can be useful when you want to use Prometheus library to instrument your code but still want to periodically export metrics to non Prometheus monitoring systems.
At moment only Counters, Gauges and Summaries are supported.
[Godoc](https://godoc.org/github.com/FZambia/eagle)
vendor/github.com/FZambia/eagle/eagle.go
Generated Vendored
-367
View File
@@ -1,367 +0,0 @@
// Package eagle provides a functionality to export Prometheus metrics
// aggregated over configured time interval. This can be useful when you
// want to use Prometheus library to instrument your code but still want
// to periodically export metrics to non Prometheus monitoring systems.
package eagle
import (
"fmt"
"math"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
)
const defaultQuantileSep = "."
// MetricType is a type for different supported metric types.
type MetricType string
// Various metric types eagle supports.
const (
MetricTypeCounter = "counter"
MetricTypeGauge = "gauge"
MetricTypeSummary = "summary"
)
// Eagle allows to periodically export Prometheus metrics
// aggregated over configured time interval.
type Eagle struct {
mu sync.RWMutex
gatherer prometheus.Gatherer
interval time.Duration
sink chan<- Metrics
quantileSep string
values map[string]float64
deltas map[string]float64
closeOnce sync.Once
closeCh chan struct{}
}
// Config of Eagle instance.
type Config struct {
Gatherer prometheus.Gatherer
Interval time.Duration
Sink chan<- Metrics
QuantileSep string
}
// New creates new Eagle.
func New(c Config) *Eagle {
e := &Eagle{
gatherer: c.Gatherer,
interval: c.Interval,
sink: c.Sink,
quantileSep: defaultQuantileSep,
values: make(map[string]float64),
deltas: make(map[string]float64),
closeCh: make(chan struct{}),
}
if c.QuantileSep != "" {
e.quantileSep = c.QuantileSep
}
go e.aggregate()
return e
}
// Close closes Eagle.
func (e *Eagle) Close() error {
e.closeOnce.Do(func() {
close(e.closeCh)
})
return nil
}
// MetricValue is a concrete value of certain metric.
type MetricValue struct {
Name string `json:"name"`
Labels []string `json:"labels,omitempty"`
Value float64 `json:"value"`
}
// Metric is a single Prometheus metric that can have many MetricValue.
type Metric struct {
Type MetricType `json:"type"`
Namespace string `json:"namespace"`
Subsystem string `json:"subsystem"`
Name string `json:"name"`
Help string `json:"help,omitempty"`
Values []MetricValue `json:"values,omitempty"`
}
// Metrics represent collection of aggregated Prometheus metrics.
type Metrics struct {
Items []Metric `json:"items,omitempty"`
}
// Flatten is a helper method to flatten metrics into map[string]float64.
func (m Metrics) Flatten(sep string) map[string]float64 {
result := make(map[string]float64)
for _, item := range m.Items {
for _, metricValue := range item.Values {
parts := []string{}
if item.Namespace != "" {
parts = append(parts, item.Namespace)
}
if item.Subsystem != "" {
parts = append(parts, item.Subsystem)
}
if item.Name != "" {
parts = append(parts, item.Name)
}
if metricValue.Name != "" {
parts = append(parts, metricValue.Name)
}
parts = append(parts, metricValue.Labels...)
key := strings.Join(parts, sep)
result[key] = metricValue.Value
}
}
return result
}
type metricLabel struct {
Name string
Value string
}
func getCacheKey(name string, labels []metricLabel, suffix string) string {
key := name
path := joinLabels(labels, ".")
if path != "" {
key += "." + path
}
if suffix != "" {
key += "." + suffix
}
return key
}
func quantileString(val float64) string {
q := fmt.Sprintf("%d", int(val*1000))
if len(q) > 2 {
q = strings.TrimSuffix(q, "0")
}
return q
}
func (e *Eagle) aggregateOnce() {
mfs, err := e.gatherer.Gather()
if err != nil {
return
}
e.mu.Lock()
defer e.mu.Unlock()
for _, mf := range mfs {
typ := mf.GetType()
name := mf.GetName()
for _, m := range mf.GetMetric() {
if typ == dto.MetricType_COUNTER {
counter := m.GetCounter()
cacheKey := getCacheKey(name, getLabels(m.GetLabel()), "")
if previousVal, ok := e.values[cacheKey]; ok {
e.deltas[cacheKey] = counter.GetValue() - previousVal
} else {
e.deltas[cacheKey] = counter.GetValue()
}
e.values[cacheKey] = counter.GetValue()
} else if typ == dto.MetricType_SUMMARY {
summary := m.GetSummary()
count := summary.GetSampleCount()
sum := summary.GetSampleSum()
cacheKey := getCacheKey(name, getLabels(m.GetLabel()), "sum")
if previousVal, ok := e.values[cacheKey]; ok {
e.deltas[cacheKey] = sum - previousVal
} else {
e.deltas[cacheKey] = sum
}
e.values[cacheKey] = sum
cacheKey = getCacheKey(name, getLabels(m.GetLabel()), "count")
if previousVal, ok := e.values[cacheKey]; ok {
e.deltas[cacheKey] = float64(count) - previousVal
} else {
e.deltas[cacheKey] = float64(count)
}
e.values[cacheKey] = float64(count)
}
}
}
if e.sink != nil {
metrics, err := e.getMetrics(mfs)
if err != nil {
return
}
select {
case e.sink <- metrics:
default:
return
}
}
}
func (e *Eagle) aggregate() {
for {
select {
case <-time.After(e.interval):
e.aggregateOnce()
case <-e.closeCh:
return
}
}
}
func getLabels(pairs []*dto.LabelPair) []metricLabel {
labels := []metricLabel{}
for _, pair := range pairs {
val := pair.GetValue()
if val == "" {
continue
}
label := metricLabel{pair.GetName(), val}
labels = append(labels, label)
}
return labels
}
func flattenLabels(labels []metricLabel) []string {
l := []string{}
for _, label := range labels {
l = append(l, label.Name)
l = append(l, label.Value)
}
return l
}
func joinLabels(labels []metricLabel, sep string) string {
chunks := []string{}
for _, lbl := range labels {
chunks = append(chunks, lbl.Name)
chunks = append(chunks, lbl.Value)
}
if len(chunks) == 0 {
return ""
}
return strings.Join(chunks, sep)
}
// Lock must be held outside.
func (e *Eagle) getMetrics(mfs []*dto.MetricFamily) (Metrics, error) {
metrics := Metrics{
Items: make([]Metric, 0),
}
for _, mf := range mfs {
typ := mf.GetType()
name := mf.GetName()
parts := strings.SplitN(name, "_", 3)
var namespace, subsystem, shortName string
if len(parts) == 3 {
namespace, subsystem, shortName = parts[0], parts[1], parts[2]
} else {
namespace, subsystem = parts[0], parts[1]
}
metric := Metric{
Namespace: namespace,
Subsystem: subsystem,
Name: shortName,
Help: mf.GetHelp(),
Values: make([]MetricValue, 0),
}
for _, m := range mf.GetMetric() {
if typ == dto.MetricType_COUNTER {
metric.Type = MetricTypeCounter
counter := m.GetCounter()
labels := getLabels(m.GetLabel())
cacheKey := getCacheKey(name, labels, "")
deltaVal, ok := e.deltas[cacheKey]
if !ok {
deltaVal = counter.GetValue()
}
value := MetricValue{
Name: "",
Labels: flattenLabels(labels),
Value: deltaVal,
}
metric.Values = append(metric.Values, value)
} else if typ == dto.MetricType_GAUGE {
metric.Type = MetricTypeGauge
gauge := m.GetGauge()
labels := getLabels(m.GetLabel())
chunks := []string{}
for _, lbl := range labels {
chunks = append(chunks, lbl.Name)
chunks = append(chunks, lbl.Value)
}
value := MetricValue{
Name: "",
Labels: flattenLabels(labels),
Value: gauge.GetValue(),
}
metric.Values = append(metric.Values, value)
} else if typ == dto.MetricType_SUMMARY {
metric.Type = MetricTypeSummary
summary := m.GetSummary()
count := summary.GetSampleCount()
sum := summary.GetSampleSum()
quantiles := summary.GetQuantile()
labels := getLabels(m.GetLabel())
deltaVal, ok := e.deltas[getCacheKey(name, labels, "sum")]
if !ok {
deltaVal = sum
}
value := MetricValue{
Name: "sum",
Labels: flattenLabels(labels),
Value: deltaVal,
}
metric.Values = append(metric.Values, value)
deltaVal, ok = e.deltas[getCacheKey(name, labels, "count")]
if !ok {
deltaVal = float64(count)
}
value = MetricValue{
Name: "count",
Labels: flattenLabels(labels),
Value: deltaVal,
}
metric.Values = append(metric.Values, value)
for _, quantile := range quantiles {
value := quantile.GetValue()
if math.IsNaN(value) {
continue
}
v := MetricValue{
Name: "quantile" + e.quantileSep + quantileString(quantile.GetQuantile()),
Labels: flattenLabels(labels),
Value: quantile.GetValue(),
}
metric.Values = append(metric.Values, v)
}
} else {
continue
}
}
metrics.Items = append(metrics.Items, metric)
}
return metrics, nil
}
// Export actual Metrics once.
func (e *Eagle) Export() (Metrics, error) {
mfs, err := e.gatherer.Gather()
if err != nil {
return Metrics{}, err
}
e.mu.Lock()
defer e.mu.Unlock()
return e.getMetrics(mfs)
}
vendor/github.com/FZambia/eagle/go.mod
Generated Vendored
-9
View File
@@ -1,9 +0,0 @@
module github.com/FZambia/eagle
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v0.9.2
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/stretchr/testify v1.2.2
)
vendor/github.com/FZambia/eagle/go.sum
Generated Vendored
-22
View File
@@ -1,22 +0,0 @@
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
vendor/github.com/FZambia/sentinel/.gitignore
Generated
-24
View File
@@ -1,24 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
vendor/github.com/FZambia/sentinel/LICENSE
Generated Vendored
-201
View File
@@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
vendor/github.com/FZambia/sentinel/README.md
Generated Vendored
-39
View File
@@ -1,39 +0,0 @@
go-sentinel
===========
Redis Sentinel support for [redigo](https://github.com/gomodule/redigo) library.
Documentation
-------------
- [API Reference](http://godoc.org/github.com/FZambia/sentinel)
Alternative solution
--------------------
You can alternatively configure Haproxy between your application and Redis to proxy requests to Redis master instance if you only need HA:
```
listen redis
server redis-01 127.0.0.1:6380 check port 6380 check inter 2s weight 1 inter 2s downinter 5s rise 10 fall 2 on-marked-down shutdown-sessions on-marked-up shutdown-backup-sessions
server redis-02 127.0.0.1:6381 check port 6381 check inter 2s weight 1 inter 2s downinter 5s rise 10 fall 2 backup
bind *:6379
mode tcp
option tcpka
option tcplog
option tcp-check
tcp-check send PING\r\n
tcp-check expect string +PONG
tcp-check send info\ replication\r\n
tcp-check expect string role:master
tcp-check send QUIT\r\n
tcp-check expect string +OK
balance roundrobin
```
This way you don't need to use this library.
License
-------
Library is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).
vendor/github.com/FZambia/sentinel/sentinel.go
Generated Vendored
-421
View File
@@ -1,421 +0,0 @@
package sentinel
import (
"errors"
"fmt"
"net"
"strings"
"sync"
"time"
"github.com/gomodule/redigo/redis"
)
// Sentinel provides a way to add high availability (HA) to Redis Pool using
// preconfigured addresses of Sentinel servers and name of master which Sentinels
// monitor. It works with Redis >= 2.8.12 (mostly because of ROLE command that
// was introduced in that version, it's possible though to support old versions
// using INFO command).
//
// Example of the simplest usage to contact master "mymaster":
//
// func newSentinelPool() *redis.Pool {
// sntnl := &sentinel.Sentinel{
// Addrs: []string{":26379", ":26380", ":26381"},
// MasterName: "mymaster",
// Dial: func(addr string) (redis.Conn, error) {
// timeout := 500 * time.Millisecond
// c, err := redis.DialTimeout("tcp", addr, timeout, timeout, timeout)
// if err != nil {
// return nil, err
// }
// return c, nil
// },
// }
// return &redis.Pool{
// MaxIdle: 3,
// MaxActive: 64,
// Wait: true,
// IdleTimeout: 240 * time.Second,
// Dial: func() (redis.Conn, error) {
// masterAddr, err := sntnl.MasterAddr()
// if err != nil {
// return nil, err
// }
// c, err := redis.Dial("tcp", masterAddr)
// if err != nil {
// return nil, err
// }
// return c, nil
// },
// TestOnBorrow: func(c redis.Conn, t time.Time) error {
// if !sentinel.TestRole(c, "master") {
// return errors.New("Role check failed")
// } else {
// return nil
// }
// },
// }
// }
type Sentinel struct {
// Addrs is a slice with known Sentinel addresses.
Addrs []string
// MasterName is a name of Redis master Sentinel servers monitor.
MasterName string
// Dial is a user supplied function to connect to Sentinel on given address. This
// address will be chosen from Addrs slice.
// Note that as per the redis-sentinel client guidelines, a timeout is mandatory
// while connecting to Sentinels, and should not be set to 0.
Dial func(addr string) (redis.Conn, error)
// Pool is a user supplied function returning custom connection pool to Sentinel.
// This can be useful to tune options if you are not satisfied with what default
// Sentinel pool offers. See defaultPool() method for default pool implementation.
// In most cases you only need to provide Dial function and let this be nil.
Pool func(addr string) *redis.Pool
mu sync.RWMutex
pools map[string]*redis.Pool
addr string
}
// NoSentinelsAvailable is returned when all sentinels in the list are exhausted
// (or none configured), and contains the last error returned by Dial (which
// may be nil)
type NoSentinelsAvailable struct {
lastError error
}
func (ns NoSentinelsAvailable) Error() string {
if ns.lastError != nil {
return fmt.Sprintf("redigo: no sentinels available; last error: %s", ns.lastError.Error())
}
return fmt.Sprintf("redigo: no sentinels available")
}
// putToTop puts Sentinel address to the top of address list - this means
// that all next requests will use Sentinel on this address first.
//
// From Sentinel guidelines:
//
// The first Sentinel replying to the client request should be put at the
// start of the list, so that at the next reconnection, we'll try first
// the Sentinel that was reachable in the previous connection attempt,
// minimizing latency.
//
// Lock must be held by caller.
func (s *Sentinel) putToTop(addr string) {
addrs := s.Addrs
if addrs[0] == addr {
// Already on top.
return
}
newAddrs := []string{addr}
for _, a := range addrs {
if a == addr {
continue
}
newAddrs = append(newAddrs, a)
}
s.Addrs = newAddrs
}
// putToBottom puts Sentinel address to the bottom of address list.
// We call this method internally when see that some Sentinel failed to answer
// on application request so next time we start with another one.
//
// Lock must be held by caller.
func (s *Sentinel) putToBottom(addr string) {
addrs := s.Addrs
if addrs[len(addrs)-1] == addr {
// Already on bottom.
return
}
newAddrs := []string{}
for _, a := range addrs {
if a == addr {
continue
}
newAddrs = append(newAddrs, a)
}
newAddrs = append(newAddrs, addr)
s.Addrs = newAddrs
}
// defaultPool returns a connection pool to one Sentinel. This allows
// us to call concurrent requests to Sentinel using connection Do method.
func (s *Sentinel) defaultPool(addr string) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
MaxActive: 10,
Wait: true,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
return s.Dial(addr)
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
func (s *Sentinel) get(addr string) redis.Conn {
pool := s.poolForAddr(addr)
return pool.Get()
}
func (s *Sentinel) poolForAddr(addr string) *redis.Pool {
s.mu.Lock()
if s.pools == nil {
s.pools = make(map[string]*redis.Pool)
}
pool, ok := s.pools[addr]
if ok {
s.mu.Unlock()
return pool
}
s.mu.Unlock()
newPool := s.newPool(addr)
s.mu.Lock()
p, ok := s.pools[addr]
if ok {
s.mu.Unlock()
return p
}
s.pools[addr] = newPool
s.mu.Unlock()
return newPool
}
func (s *Sentinel) newPool(addr string) *redis.Pool {
if s.Pool != nil {
return s.Pool(addr)
}
return s.defaultPool(addr)
}
// close connection pool to Sentinel.
// Lock must be hold by caller.
func (s *Sentinel) close() {
if s.pools != nil {
for _, pool := range s.pools {
pool.Close()
}
}
s.pools = nil
}
func (s *Sentinel) doUntilSuccess(f func(redis.Conn) (interface{}, error)) (interface{}, error) {
s.mu.RLock()
addrs := s.Addrs
s.mu.RUnlock()
var lastErr error
for _, addr := range addrs {
conn := s.get(addr)
reply, err := f(conn)
conn.Close()
if err != nil {
lastErr = err
s.mu.Lock()
s.putToBottom(addr)
s.mu.Unlock()
continue
}
s.putToTop(addr)
return reply, nil
}
return nil, NoSentinelsAvailable{lastError: lastErr}
}
// MasterAddr returns an address of current Redis master instance.
func (s *Sentinel) MasterAddr() (string, error) {
res, err := s.doUntilSuccess(func(c redis.Conn) (interface{}, error) {
return queryForMaster(c, s.MasterName)
})
if err != nil {
return "", err
}
return res.(string), nil
}
// SlaveAddrs returns a slice with known slave addresses of current master instance.
func (s *Sentinel) SlaveAddrs() ([]string, error) {
res, err := s.doUntilSuccess(func(c redis.Conn) (interface{}, error) {
return queryForSlaveAddrs(c, s.MasterName)
})
if err != nil {
return nil, err
}
return res.([]string), nil
}
// Slave represents a Redis slave instance which is known by Sentinel.
type Slave struct {
ip string
port string
flags string
}
// Addr returns an address of slave.
func (s *Slave) Addr() string {
return net.JoinHostPort(s.ip, s.port)
}
// Available returns if slave is in working state at moment based on information in slave flags.
func (s *Slave) Available() bool {
return !strings.Contains(s.flags, "disconnected") && !strings.Contains(s.flags, "s_down")
}
// Slaves returns a slice with known slaves of master instance.
func (s *Sentinel) Slaves() ([]*Slave, error) {
res, err := s.doUntilSuccess(func(c redis.Conn) (interface{}, error) {
return queryForSlaves(c, s.MasterName)
})
if err != nil {
return nil, err
}
return res.([]*Slave), nil
}
// SentinelAddrs returns a slice of known Sentinel addresses Sentinel server aware of.
func (s *Sentinel) SentinelAddrs() ([]string, error) {
res, err := s.doUntilSuccess(func(c redis.Conn) (interface{}, error) {
return queryForSentinels(c, s.MasterName)
})
if err != nil {
return nil, err
}
return res.([]string), nil
}
// Discover allows to update list of known Sentinel addresses. From docs:
//
// A client may update its internal list of Sentinel nodes following this procedure:
// 1) Obtain a list of other Sentinels for this master using the command SENTINEL sentinels <master-name>.
// 2) Add every ip:port pair not already existing in our list at the end of the list.
func (s *Sentinel) Discover() error {
addrs, err := s.SentinelAddrs()
if err != nil {
return err
}
s.mu.Lock()
for _, addr := range addrs {
if !stringInSlice(addr, s.Addrs) {
s.Addrs = append(s.Addrs, addr)
}
}
s.mu.Unlock()
return nil
}
// Close closes current connection to Sentinel.
func (s *Sentinel) Close() error {
s.mu.Lock()
s.close()
s.mu.Unlock()
return nil
}
// TestRole wraps GetRole in a test to verify if the role matches an expected
// role string. If there was any error in querying the supplied connection,
// the function returns false. Works with Redis >= 2.8.12.
// It's not goroutine safe, but if you call this method on pooled connections
// then you are OK.
func TestRole(c redis.Conn, expectedRole string) bool {
role, err := getRole(c)
if err != nil || role != expectedRole {
return false
}
return true
}
// getRole is a convenience function supplied to query an instance (master or
// slave) for its role. It attempts to use the ROLE command introduced in
// redis 2.8.12.
func getRole(c redis.Conn) (string, error) {
res, err := c.Do("ROLE")
if err != nil {
return "", err
}
rres, ok := res.([]interface{})
if ok {
return redis.String(rres[0], nil)
}
return "", errors.New("redigo: can not transform ROLE reply to string")
}
func queryForMaster(conn redis.Conn, masterName string) (string, error) {
res, err := redis.Strings(conn.Do("SENTINEL", "get-master-addr-by-name", masterName))
if err != nil {
return "", err
}
if len(res) < 2 {
return "", errors.New("redigo: malformed get-master-addr-by-name reply")
}
masterAddr := net.JoinHostPort(res[0], res[1])
return masterAddr, nil
}
func queryForSlaveAddrs(conn redis.Conn, masterName string) ([]string, error) {
slaves, err := queryForSlaves(conn, masterName)
if err != nil {
return nil, err
}
slaveAddrs := make([]string, 0)
for _, slave := range slaves {
slaveAddrs = append(slaveAddrs, slave.Addr())
}
return slaveAddrs, nil
}
func queryForSlaves(conn redis.Conn, masterName string) ([]*Slave, error) {
res, err := redis.Values(conn.Do("SENTINEL", "slaves", masterName))
if err != nil {
return nil, err
}
slaves := make([]*Slave, 0)
for _, a := range res {
sm, err := redis.StringMap(a, err)
if err != nil {
return slaves, err
}
slave := &Slave{
ip: sm["ip"],
port: sm["port"],
flags: sm["flags"],
}
slaves = append(slaves, slave)
}
return slaves, nil
}
func queryForSentinels(conn redis.Conn, masterName string) ([]string, error) {
res, err := redis.Values(conn.Do("SENTINEL", "sentinels", masterName))
if err != nil {
return nil, err
}
sentinels := make([]string, 0)
for _, a := range res {
sm, err := redis.StringMap(a, err)
if err != nil {
return sentinels, err
}
sentinels = append(sentinels, fmt.Sprintf("%s:%s", sm["ip"], sm["port"]))
}
return sentinels, nil
}
func stringInSlice(str string, slice []string) bool {
for _, s := range slice {
if s == str {
return true
}
}
return false
}
vendor/github.com/PuerkitoBio/purell/.gitignore
Generated
-5
View File
@@ -1,5 +0,0 @@
*.sublime-*
.DS_Store
*.swp
*.swo
tags
vendor/github.com/PuerkitoBio/purell/.travis.yml
Generated Vendored
-12
View File
@@ -1,12 +0,0 @@
language: go
go:
- 1.4.x
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- "1.10.x"
- "1.11.x"
- tip
vendor/github.com/PuerkitoBio/purell/LICENSE
Generated Vendored
-12
View File
@@ -1,12 +0,0 @@
Copyright (c) 2012, Martin Angers
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/PuerkitoBio/purell/README.md
Generated Vendored
-188
View File
@@ -1,188 +0,0 @@
# Purell
Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know...
Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc].
[![build status](https://travis-ci.org/PuerkitoBio/purell.svg?branch=master)](http://travis-ci.org/PuerkitoBio/purell)
## Install
`go get github.com/PuerkitoBio/purell`
## Changelog
* **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor).
* **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121).
* **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich).
* **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]).
* **v0.2.0** : Add benchmarks, Attempt IDN support.
* **v0.1.0** : Initial release.
## Examples
From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."):
```go
package purell
import (
"fmt"
"net/url"
)
func ExampleNormalizeURLString() {
if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/",
FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil {
panic(err)
} else {
fmt.Print(normalized)
}
// Output: http://somewebsite.com:80/Amazing%3F/url/
}
func ExampleMustNormalizeURLString() {
normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/",
FlagsUnsafeGreedy)
fmt.Print(normalized)
// Output: http://somewebsite.com/Amazing%FA/url
}
func ExampleNormalizeURL() {
if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil {
panic(err)
} else {
normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment)
fmt.Print(normalized)
}
// Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0
}
```
## API
As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags:
```go
const (
// Safe normalizations
FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
FlagLowercaseHost // http://HOST -> http://host
FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF
FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA
FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$
FlagRemoveDefaultPort // http://host:80 -> http://host
FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path
// Usually safe normalizations
FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c
// Unsafe normalizations
FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/
FlagRemoveFragment // http://host/path#fragment -> http://host/path
FlagForceHTTP // https://host -> http://host
FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
FlagRemoveWWW // http://www.host/ -> http://host/
FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
// Normalizations not in the wikipedia article, required to cover tests cases
// submitted by jehiah
FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147
FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147
FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147
FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path
// Convenience set of safe normalizations
FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
// Convenience set of usually safe normalizations (includes FlagsSafe)
FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
// Convenience set of all available flags
FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
)
```
For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set.
The [full godoc reference is available on gopkgdoc][godoc].
Some things to note:
* `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it.
* The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*):
- %24 -> $
- %26 -> &
- %2B-%3B -> +,-./0123456789:;
- %3D -> =
- %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ
- %5F -> _
- %61-%7A -> abcdefghijklmnopqrstuvwxyz
- %7E -> ~
* When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization).
* The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell.
* The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object.
### Safe vs Usually Safe vs Unsafe
Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between.
Consider the following URL:
`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
Normalizing with the `FlagsSafe` gives:
`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
With the `FlagsUsuallySafeGreedy`:
`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid`
And with `FlagsUnsafeGreedy`:
`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3`
## TODOs
* Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`.
## Thanks / Contributions
@rogpeppe
@jehiah
@opennota
@pchristopher1275
@zenovich
@beeker1121
## License
The [BSD 3-Clause license][bsd].
[bsd]: http://opensource.org/licenses/BSD-3-Clause
[wiki]: http://en.wikipedia.org/wiki/URL_normalization
[rfc]: http://tools.ietf.org/html/rfc3986#section-6
[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell
[pr5]: https://github.com/PuerkitoBio/purell/pull/5
[iss7]: https://github.com/PuerkitoBio/purell/issues/7
vendor/github.com/PuerkitoBio/purell/purell.go
Generated Vendored
-379
View File
@@ -1,379 +0,0 @@
/*
Package purell offers URL normalization as described on the wikipedia page:
http://en.wikipedia.org/wiki/URL_normalization
*/
package purell
import (
"bytes"
"fmt"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"github.com/PuerkitoBio/urlesc"
"golang.org/x/net/idna"
"golang.org/x/text/unicode/norm"
"golang.org/x/text/width"
)
// A set of normalization flags determines how a URL will
// be normalized.
type NormalizationFlags uint
const (
// Safe normalizations
FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
FlagLowercaseHost // http://HOST -> http://host
FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF
FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA
FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$
FlagRemoveDefaultPort // http://host:80 -> http://host
FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path
// Usually safe normalizations
FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c
// Unsafe normalizations
FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/
FlagRemoveFragment // http://host/path#fragment -> http://host/path
FlagForceHTTP // https://host -> http://host
FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
FlagRemoveWWW // http://www.host/ -> http://host/
FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
// Normalizations not in the wikipedia article, required to cover tests cases
// submitted by jehiah
FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147
FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147
FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147
FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path
// Convenience set of safe normalizations
FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
// Convenience set of usually safe normalizations (includes FlagsSafe)
FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
// Convenience set of all available flags
FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
)
const (
defaultHttpPort = ":80"
defaultHttpsPort = ":443"
)
// Regular expressions used by the normalizations
var rxPort = regexp.MustCompile(`(:\d+)/?$`)
var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
var rxDupSlashes = regexp.MustCompile(`/{2,}`)
var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
var rxEmptyPort = regexp.MustCompile(`:+$`)
// Map of flags to implementation function.
// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
// Since maps have undefined traversing order, make a slice of ordered keys
var flagsOrder = []NormalizationFlags{
FlagLowercaseScheme,
FlagLowercaseHost,
FlagRemoveDefaultPort,
FlagRemoveDirectoryIndex,
FlagRemoveDotSegments,
FlagRemoveFragment,
FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
FlagRemoveDuplicateSlashes,
FlagRemoveWWW,
FlagAddWWW,
FlagSortQuery,
FlagDecodeDWORDHost,
FlagDecodeOctalHost,
FlagDecodeHexHost,
FlagRemoveUnnecessaryHostDots,
FlagRemoveEmptyPortSeparator,
FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
FlagAddTrailingSlash,
}
// ... and then the map, where order is unimportant
var flags = map[NormalizationFlags]func(*url.URL){
FlagLowercaseScheme: lowercaseScheme,
FlagLowercaseHost: lowercaseHost,
FlagRemoveDefaultPort: removeDefaultPort,
FlagRemoveDirectoryIndex: removeDirectoryIndex,
FlagRemoveDotSegments: removeDotSegments,
FlagRemoveFragment: removeFragment,
FlagForceHTTP: forceHTTP,
FlagRemoveDuplicateSlashes: removeDuplicateSlashes,
FlagRemoveWWW: removeWWW,
FlagAddWWW: addWWW,
FlagSortQuery: sortQuery,
FlagDecodeDWORDHost: decodeDWORDHost,
FlagDecodeOctalHost: decodeOctalHost,
FlagDecodeHexHost: decodeHexHost,
FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator,
FlagRemoveTrailingSlash: removeTrailingSlash,
FlagAddTrailingSlash: addTrailingSlash,
}
// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
// It takes an URL string as input, as well as the normalization flags.
func MustNormalizeURLString(u string, f NormalizationFlags) string {
result, e := NormalizeURLString(u, f)
if e != nil {
panic(e)
}
return result
}
// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
// It takes an URL string as input, as well as the normalization flags.
func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
parsed, err := url.Parse(u)
if err != nil {
return "", err
}
if f&FlagLowercaseHost == FlagLowercaseHost {
parsed.Host = strings.ToLower(parsed.Host)
}
// The idna package doesn't fully conform to RFC 5895
// (https://tools.ietf.org/html/rfc5895), so we do it here.
// Taken from Go 1.8 cycle source, courtesy of bradfitz.
// TODO: Remove when (if?) idna package conforms to RFC 5895.
parsed.Host = width.Fold.String(parsed.Host)
parsed.Host = norm.NFC.String(parsed.Host)
if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
return "", err
}
return NormalizeURL(parsed, f), nil
}
// NormalizeURL returns the normalized string.
// It takes a parsed URL object as input, as well as the normalization flags.
func NormalizeURL(u *url.URL, f NormalizationFlags) string {
for _, k := range flagsOrder {
if f&k == k {
flags[k](u)
}
}
return urlesc.Escape(u)
}
func lowercaseScheme(u *url.URL) {
if len(u.Scheme) > 0 {
u.Scheme = strings.ToLower(u.Scheme)
}
}
func lowercaseHost(u *url.URL) {
if len(u.Host) > 0 {
u.Host = strings.ToLower(u.Host)
}
}
func removeDefaultPort(u *url.URL) {
if len(u.Host) > 0 {
scheme := strings.ToLower(u.Scheme)
u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
return ""
}
return val
})
}
}
func removeTrailingSlash(u *url.URL) {
if l := len(u.Path); l > 0 {
if strings.HasSuffix(u.Path, "/") {
u.Path = u.Path[:l-1]
}
} else if l = len(u.Host); l > 0 {
if strings.HasSuffix(u.Host, "/") {
u.Host = u.Host[:l-1]
}
}
}
func addTrailingSlash(u *url.URL) {
if l := len(u.Path); l > 0 {
if !strings.HasSuffix(u.Path, "/") {
u.Path += "/"
}
} else if l = len(u.Host); l > 0 {
if !strings.HasSuffix(u.Host, "/") {
u.Host += "/"
}
}
}
func removeDotSegments(u *url.URL) {
if len(u.Path) > 0 {
var dotFree []string
var lastIsDot bool
sections := strings.Split(u.Path, "/")
for _, s := range sections {
if s == ".." {
if len(dotFree) > 0 {
dotFree = dotFree[:len(dotFree)-1]
}
} else if s != "." {
dotFree = append(dotFree, s)
}
lastIsDot = (s == "." || s == "..")
}
// Special case if host does not end with / and new path does not begin with /
u.Path = strings.Join(dotFree, "/")
if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
u.Path = "/" + u.Path
}
// Special case if the last segment was a dot, make sure the path ends with a slash
if lastIsDot && !strings.HasSuffix(u.Path, "/") {
u.Path += "/"
}
}
}
func removeDirectoryIndex(u *url.URL) {
if len(u.Path) > 0 {
u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
}
}
func removeFragment(u *url.URL) {
u.Fragment = ""
}
func forceHTTP(u *url.URL) {
if strings.ToLower(u.Scheme) == "https" {
u.Scheme = "http"
}
}
func removeDuplicateSlashes(u *url.URL) {
if len(u.Path) > 0 {
u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
}
}
func removeWWW(u *url.URL) {
if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
u.Host = u.Host[4:]
}
}
func addWWW(u *url.URL) {
if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
u.Host = "www." + u.Host
}
}
func sortQuery(u *url.URL) {
q := u.Query()
if len(q) > 0 {
arKeys := make([]string, len(q))
i := 0
for k := range q {
arKeys[i] = k
i++
}
sort.Strings(arKeys)
buf := new(bytes.Buffer)
for _, k := range arKeys {
sort.Strings(q[k])
for _, v := range q[k] {
if buf.Len() > 0 {
buf.WriteRune('&')
}
buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
}
}
// Rebuild the raw query string
u.RawQuery = buf.String()
}
}
func decodeDWORDHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
var parts [4]int64
dword, _ := strconv.ParseInt(matches[1], 10, 0)
for i, shift := range []uint{24, 16, 8, 0} {
parts[i] = dword >> shift & 0xFF
}
u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
}
}
}
func decodeOctalHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
var parts [4]int64
for i := 1; i <= 4; i++ {
parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
}
u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
}
}
}
func decodeHexHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
// Conversion is safe because of regex validation
parsed, _ := strconv.ParseInt(matches[1], 16, 0)
// Set host as DWORD (base 10) encoded host
u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
// The rest is the same as decoding a DWORD host
decodeDWORDHost(u)
}
}
}
func removeUnncessaryHostDots(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
// Trim the leading and trailing dots
u.Host = strings.Trim(matches[1], ".")
if len(matches) > 2 {
u.Host += matches[2]
}
}
}
}
func removeEmptyPortSeparator(u *url.URL) {
if len(u.Host) > 0 {
u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
}
}
vendor/github.com/PuerkitoBio/urlesc/.travis.yml
Generated Vendored
-15
View File
@@ -1,15 +0,0 @@
language: go
go:
- 1.4.x
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x
- tip
install:
- go build .
script:
- go test -v
vendor/github.com/PuerkitoBio/urlesc/LICENSE
Generated Vendored
-27
View File
@@ -1,27 +0,0 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/PuerkitoBio/urlesc/README.md
Generated Vendored
-16
View File
@@ -1,16 +0,0 @@
urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc)
======
Package urlesc implements query escaping as per RFC 3986.
It contains some parts of the net/url package, modified so as to allow
some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)).
## Install
go get github.com/PuerkitoBio/urlesc
## License
Go license (BSD-3-Clause)
vendor/github.com/PuerkitoBio/urlesc/urlesc.go
Generated Vendored
-180
View File
@@ -1,180 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package urlesc implements query escaping as per RFC 3986.
// It contains some parts of the net/url package, modified so as to allow
// some reserved characters incorrectly escaped by net/url.
// See https://github.com/golang/go/issues/5684
package urlesc
import (
"bytes"
"net/url"
"strings"
)
type encoding int
const (
encodePath encoding = 1 + iota
encodeUserPassword
encodeQueryComponent
encodeFragment
)
// Return true if the specified character should be escaped when
// appearing in a URL string, according to RFC 3986.
func shouldEscape(c byte, mode encoding) bool {
// §2.3 Unreserved characters (alphanum)
if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
return false
}
switch c {
case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
return false
// §2.2 Reserved characters (reserved)
case ':', '/', '?', '#', '[', ']', '@', // gen-delims
'!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
// Different sections of the URL allow a few of
// the reserved characters to appear unescaped.
switch mode {
case encodePath: // §3.3
// The RFC allows sub-delims and : @.
// '/', '[' and ']' can be used to assign meaning to individual path
// segments. This package only manipulates the path as a whole,
// so we allow those as well. That leaves only ? and # to escape.
return c == '?' || c == '#'
case encodeUserPassword: // §3.2.1
// The RFC allows : and sub-delims in
// userinfo. The parsing of userinfo treats ':' as special so we must escape
// all the gen-delims.
return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
case encodeQueryComponent: // §3.4
// The RFC allows / and ?.
return c != '/' && c != '?'
case encodeFragment: // §4.1
// The RFC text is silent but the grammar allows
// everything, so escape nothing but #
return c == '#'
}
}
// Everything else must be escaped.
return true
}
// QueryEscape escapes the string so it can be safely placed
// inside a URL query.
func QueryEscape(s string) string {
return escape(s, encodeQueryComponent)
}
func escape(s string, mode encoding) string {
spaceCount, hexCount := 0, 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c, mode) {
if c == ' ' && mode == encodeQueryComponent {
spaceCount++
} else {
hexCount++
}
}
}
if spaceCount == 0 && hexCount == 0 {
return s
}
t := make([]byte, len(s)+2*hexCount)
j := 0
for i := 0; i < len(s); i++ {
switch c := s[i]; {
case c == ' ' && mode == encodeQueryComponent:
t[j] = '+'
j++
case shouldEscape(c, mode):
t[j] = '%'
t[j+1] = "0123456789ABCDEF"[c>>4]
t[j+2] = "0123456789ABCDEF"[c&15]
j += 3
default:
t[j] = s[i]
j++
}
}
return string(t)
}
var uiReplacer = strings.NewReplacer(
"%21", "!",
"%27", "'",
"%28", "(",
"%29", ")",
"%2A", "*",
)
// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
func unescapeUserinfo(s string) string {
return uiReplacer.Replace(s)
}
// Escape reassembles the URL into a valid URL string.
// The general form of the result is one of:
//
// scheme:opaque
// scheme://userinfo@host/path?query#fragment
//
// If u.Opaque is non-empty, String uses the first form;
// otherwise it uses the second form.
//
// In the second form, the following rules apply:
// - if u.Scheme is empty, scheme: is omitted.
// - if u.User is nil, userinfo@ is omitted.
// - if u.Host is empty, host/ is omitted.
// - if u.Scheme and u.Host are empty and u.User is nil,
// the entire scheme://userinfo@host/ is omitted.
// - if u.Host is non-empty and u.Path begins with a /,
// the form host/path does not add its own /.
// - if u.RawQuery is empty, ?query is omitted.
// - if u.Fragment is empty, #fragment is omitted.
func Escape(u *url.URL) string {
var buf bytes.Buffer
if u.Scheme != "" {
buf.WriteString(u.Scheme)
buf.WriteByte(':')
}
if u.Opaque != "" {
buf.WriteString(u.Opaque)
} else {
if u.Scheme != "" || u.Host != "" || u.User != nil {
buf.WriteString("//")
if ui := u.User; ui != nil {
buf.WriteString(unescapeUserinfo(ui.String()))
buf.WriteByte('@')
}
if h := u.Host; h != "" {
buf.WriteString(h)
}
}
if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
buf.WriteByte('/')
}
buf.WriteString(escape(u.Path, encodePath))
}
if u.RawQuery != "" {
buf.WriteByte('?')
buf.WriteString(u.RawQuery)
}
if u.Fragment != "" {
buf.WriteByte('#')
buf.WriteString(escape(u.Fragment, encodeFragment))
}
return buf.String()
}
vendor/github.com/apache/arrow/go/arrow/.editorconfig
Generated Vendored
-21
View File
@@ -1,21 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
root = true
[*.tmpl]
indent_style = tab
indent_size = 4
vendor/github.com/apache/arrow/go/arrow/.gitignore
Generated
-35
View File
@@ -1,35 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
### Go template
# Binaries for programs and plugins
*.exe
*.dll
*.so
*.dylib
*.o
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
bin/
vendor/
vendor/github.com/apache/arrow/go/arrow/Gopkg.lock
Generated Vendored
-44
View File
@@ -1,44 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = ""
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
version = "v0.8.1"
[[projects]]
digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = ""
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:2d0dc026c4aef5e2f3a0e06a4dabe268b840d8f63190cf6894e02134a03f52c5"
name = "github.com/stretchr/testify"
packages = ["assert"]
pruneopts = ""
revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
version = "v1.2.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/pkg/errors",
"github.com/stretchr/testify/assert",
]
solver-name = "gps-cdcl"
solver-version = 1
vendor/github.com/apache/arrow/go/arrow/Gopkg.toml
Generated Vendored
-23
View File
@@ -1,23 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.0"
[[constraint]]
name = "github.com/pkg/errors"
version = "0.8.1"
vendor/github.com/apache/arrow/go/arrow/LICENSE.txt
Generated Vendored
-2015
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/apache/arrow/go/arrow/Makefile
Generated Vendored
-54
View File
@@ -1,54 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO_BUILD=go build
GO_GEN=go generate
GO_TEST?=go test
GOPATH=$(realpath ../../../../../..)
GO_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
SOURCES_NO_VENDOR := $(shell find . -path ./vendor -prune -o -name "*.go" -not -name '*_test.go' -print)
.PHONEY: test bench assembly generate
assembly:
@$(MAKE) -C memory assembly
@$(MAKE) -C math assembly
generate: bin/tmpl
bin/tmpl -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen_test.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
bin/tmpl -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl
@$(MAKE) -C math generate
fmt: $(SOURCES_NO_VENDOR)
goimports -w $^
bench: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) -bench=. -run=- ./...
bench-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' -bench=. -run=- ./...
test: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) ./...
test-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' ./...
bin/tmpl: _tools/tmpl/main.go
$(GO_BUILD) -o $@ ./_tools/tmpl
vendor/github.com/apache/arrow/go/arrow/array/array.go
Generated Vendored
-208
View File
@@ -1,208 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
)
// A type which satisfies array.Interface represents an immutable sequence of values.
type Interface interface {
// DataType returns the type metadata for this instance.
DataType() arrow.DataType
// NullN returns the number of null values in the array.
NullN() int
// NullBitmapBytes returns a byte slice of the validity bitmap.
NullBitmapBytes() []byte
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsNull(i int) bool
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsValid(i int) bool
Data() *Data
// Len returns the number of elements in the array.
Len() int
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
Release()
}
const (
// UnknownNullCount specifies the NullN should be calculated from the null bitmap buffer.
UnknownNullCount = -1
)
type array struct {
refCount int64
data *Data
nullBitmapBytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (a *array) Retain() {
atomic.AddInt64(&a.refCount, 1)
}
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
func (a *array) Release() {
debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
if atomic.AddInt64(&a.refCount, -1) == 0 {
a.data.Release()
a.data, a.nullBitmapBytes = nil, nil
}
}
// DataType returns the type metadata for this instance.
func (a *array) DataType() arrow.DataType { return a.data.dtype }
// NullN returns the number of null values in the array.
func (a *array) NullN() int {
if a.data.nulls < 0 {
a.data.nulls = a.data.length - bitutil.CountSetBits(a.nullBitmapBytes, a.data.offset, a.data.length)
}
return a.data.nulls
}
// NullBitmapBytes returns a byte slice of the validity bitmap.
func (a *array) NullBitmapBytes() []byte { return a.nullBitmapBytes }
func (a *array) Data() *Data { return a.data }
// Len returns the number of elements in the array.
func (a *array) Len() int { return a.data.length }
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsNull(i int) bool {
return len(a.nullBitmapBytes) != 0 && bitutil.BitIsNotSet(a.nullBitmapBytes, a.data.offset+i)
}
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsValid(i int) bool {
return len(a.nullBitmapBytes) == 0 || bitutil.BitIsSet(a.nullBitmapBytes, a.data.offset+i)
}
func (a *array) setData(data *Data) {
// Retain before releasing in case a.data is the same as data.
data.Retain()
if a.data != nil {
a.data.Release()
}
if len(data.buffers) > 0 && data.buffers[0] != nil {
a.nullBitmapBytes = data.buffers[0].Bytes()
}
a.data = data
}
func (a *array) Offset() int {
return a.data.Offset()
}
type arrayConstructorFn func(*Data) Interface
var (
makeArrayFn [32]arrayConstructorFn
)
func unsupportedArrayType(data *Data) Interface {
panic("unsupported data type: " + data.dtype.ID().String())
}
func invalidDataType(data *Data) Interface {
panic("invalid data type: " + data.dtype.ID().String())
}
// MakeFromData constructs a strongly-typed array instance from generic Data.
func MakeFromData(data *Data) Interface {
return makeArrayFn[byte(data.dtype.ID()&0x1f)](data)
}
// NewSlice constructs a zero-copy slice of the array with the indicated
// indices i and j, corresponding to array[i:j].
// The returned array must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the input array.
// NewSlice panics if j < i.
func NewSlice(arr Interface, i, j int64) Interface {
data := NewSliceData(arr.Data(), i, j)
slice := MakeFromData(data)
data.Release()
return slice
}
func init() {
makeArrayFn = [...]arrayConstructorFn{
arrow.NULL: func(data *Data) Interface { return NewNullData(data) },
arrow.BOOL: func(data *Data) Interface { return NewBooleanData(data) },
arrow.UINT8: func(data *Data) Interface { return NewUint8Data(data) },
arrow.INT8: func(data *Data) Interface { return NewInt8Data(data) },
arrow.UINT16: func(data *Data) Interface { return NewUint16Data(data) },
arrow.INT16: func(data *Data) Interface { return NewInt16Data(data) },
arrow.UINT32: func(data *Data) Interface { return NewUint32Data(data) },
arrow.INT32: func(data *Data) Interface { return NewInt32Data(data) },
arrow.UINT64: func(data *Data) Interface { return NewUint64Data(data) },
arrow.INT64: func(data *Data) Interface { return NewInt64Data(data) },
arrow.FLOAT16: func(data *Data) Interface { return NewFloat16Data(data) },
arrow.FLOAT32: func(data *Data) Interface { return NewFloat32Data(data) },
arrow.FLOAT64: func(data *Data) Interface { return NewFloat64Data(data) },
arrow.STRING: func(data *Data) Interface { return NewStringData(data) },
arrow.BINARY: func(data *Data) Interface { return NewBinaryData(data) },
arrow.FIXED_SIZE_BINARY: func(data *Data) Interface { return NewFixedSizeBinaryData(data) },
arrow.DATE32: func(data *Data) Interface { return NewDate32Data(data) },
arrow.DATE64: func(data *Data) Interface { return NewDate64Data(data) },
arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) },
arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) },
arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) },
arrow.INTERVAL: func(data *Data) Interface { return NewIntervalData(data) },
arrow.DECIMAL: func(data *Data) Interface { return NewDecimal128Data(data) },
arrow.LIST: func(data *Data) Interface { return NewListData(data) },
arrow.STRUCT: func(data *Data) Interface { return NewStructData(data) },
arrow.UNION: unsupportedArrayType,
arrow.DICTIONARY: unsupportedArrayType,
arrow.MAP: unsupportedArrayType,
arrow.EXTENSION: unsupportedArrayType,
arrow.FIXED_SIZE_LIST: func(data *Data) Interface { return NewFixedSizeListData(data) },
arrow.DURATION: func(data *Data) Interface { return NewDurationData(data) },
// invalid data types to fill out array size 2⁵-1
31: invalidDataType,
}
}
vendor/github.com/apache/arrow/go/arrow/array/binary.go
Generated Vendored
-134
View File
@@ -1,134 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"unsafe"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of variable-length binary strings.
type Binary struct {
array
valueOffsets []int32
valueBytes []byte
}
// NewBinaryData constructs a new Binary array from data.
func NewBinaryData(data *Data) *Binary {
a := &Binary{}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the slice at index i. This value should not be mutated.
func (a *Binary) Value(i int) []byte {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
idx := a.array.data.offset + i
return a.valueBytes[a.valueOffsets[idx]:a.valueOffsets[idx+1]]
}
// ValueString returns the string at index i without performing additional allocations.
// The string is only valid for the lifetime of the Binary array.
func (a *Binary) ValueString(i int) string {
b := a.Value(i)
return *(*string)(unsafe.Pointer(&b))
}
func (a *Binary) ValueOffset(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return int(a.valueOffsets[a.array.data.offset+i])
}
func (a *Binary) ValueLen(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
beg := a.array.data.offset + i
return int(a.valueOffsets[beg+1] - a.valueOffsets[beg])
}
func (a *Binary) ValueOffsets() []int32 {
beg := a.array.data.offset
end := beg + a.array.data.length + 1
return a.valueOffsets[beg:end]
}
func (a *Binary) ValueBytes() []byte {
beg := a.array.data.offset
end := beg + a.array.data.length
return a.valueBytes[a.valueOffsets[beg]:a.valueOffsets[end]]
}
func (a *Binary) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.ValueString(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Binary) setData(data *Data) {
if len(data.buffers) != 3 {
panic("len(data.buffers) != 3")
}
a.array.setData(data)
if valueData := data.buffers[2]; valueData != nil {
a.valueBytes = valueData.Bytes()
}
if valueOffsets := data.buffers[1]; valueOffsets != nil {
a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
}
}
func arrayEqualBinary(left, right *Binary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
return false
}
}
return true
}
var (
_ Interface = (*Binary)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go
Generated Vendored
-217
View File
@@ -1,217 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"math"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
const (
binaryArrayMaximumCapacity = math.MaxInt32
)
// A BinaryBuilder is used to build a Binary array using the Append methods.
type BinaryBuilder struct {
builder
dtype arrow.BinaryDataType
offsets *int32BufferBuilder
values *byteBufferBuilder
}
func NewBinaryBuilder(mem memory.Allocator, dtype arrow.BinaryDataType) *BinaryBuilder {
b := &BinaryBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
offsets: newInt32BufferBuilder(mem),
values: newByteBufferBuilder(mem),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BinaryBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.offsets != nil {
b.offsets.Release()
b.offsets = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *BinaryBuilder) Append(v []byte) {
b.Reserve(1)
b.appendNextOffset()
b.values.Append(v)
b.UnsafeAppendBoolToBitmap(true)
}
func (b *BinaryBuilder) AppendString(v string) {
b.Append([]byte(v))
}
func (b *BinaryBuilder) AppendNull() {
b.Reserve(1)
b.appendNextOffset()
b.UnsafeAppendBoolToBitmap(false)
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append(vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
// AppendStringValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append([]byte(vv))
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BinaryBuilder) Value(i int) []byte {
offsets := b.offsets.Values()
start := int(offsets[i])
var end int
if i == (b.length - 1) {
end = b.values.Len()
} else {
end = int(offsets[i+1])
}
return b.values.Bytes()[start:end]
}
func (b *BinaryBuilder) init(capacity int) {
b.builder.init(capacity)
b.offsets.resize((capacity + 1) * arrow.Int32SizeBytes)
}
// DataLen returns the number of bytes in the data array.
func (b *BinaryBuilder) DataLen() int { return b.values.length }
// DataCap returns the total number of bytes that can be stored
// without allocating additional memory.
func (b *BinaryBuilder) DataCap() int { return b.values.capacity }
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BinaryBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// ReserveData ensures there is enough space for appending n bytes
// by checking the capacity and resizing the data buffer if necessary.
func (b *BinaryBuilder) ReserveData(n int) {
if b.values.capacity < b.values.length+n {
b.values.resize(b.values.Len() + n)
}
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may be reduced.
func (b *BinaryBuilder) Resize(n int) {
b.offsets.resize((n + 1) * arrow.Int32SizeBytes)
b.builder.resize(n, b.init)
}
// NewArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewArray() Interface {
return b.NewBinaryArray()
}
// NewBinaryArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewBinaryArray() (a *Binary) {
data := b.newData()
a = NewBinaryData(data)
data.Release()
return
}
func (b *BinaryBuilder) newData() (data *Data) {
b.appendNextOffset()
offsets, values := b.offsets.Finish(), b.values.Finish()
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0)
if offsets != nil {
offsets.Release()
}
if values != nil {
values.Release()
}
b.builder.reset()
return
}
func (b *BinaryBuilder) appendNextOffset() {
numBytes := b.values.Len()
// TODO(sgc): check binaryArrayMaximumCapacity?
b.offsets.AppendValue(int32(numBytes))
}
var (
_ Builder = (*BinaryBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/boolean.go
Generated Vendored
-95
View File
@@ -1,95 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents an immutable sequence of boolean values.
type Boolean struct {
array
values []byte
}
// NewBoolean creates a boolean array from the data memory.Buffer and contains length elements.
// The nullBitmap buffer can be nil of there are no null values.
// If nulls is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer.
func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nulls int) *Boolean {
return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls, 0))
}
func NewBooleanData(data *Data) *Boolean {
a := &Boolean{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Boolean) Value(i int) bool {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return bitutil.BitIsSet(a.values, a.array.data.offset+i)
}
func (a *Boolean) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Boolean) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = vals.Bytes()
}
}
func arrayEqualBoolean(left, right *Boolean) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
var (
_ Interface = (*Boolean)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go
Generated Vendored
-165
View File
@@ -1,165 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
type BooleanBuilder struct {
builder
data *memory.Buffer
rawData []byte
}
func NewBooleanBuilder(mem memory.Allocator) *BooleanBuilder {
return &BooleanBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BooleanBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *BooleanBuilder) Append(v bool) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *BooleanBuilder) AppendByte(v byte) {
b.Reserve(1)
b.UnsafeAppend(v != 0)
}
func (b *BooleanBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *BooleanBuilder) UnsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
if v {
bitutil.SetBit(b.rawData, b.length)
} else {
bitutil.ClearBit(b.rawData, b.length)
}
b.length++
}
func (b *BooleanBuilder) AppendValues(v []bool, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for i, vv := range v {
bitutil.SetBitTo(b.rawData, b.length+i, vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BooleanBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.BooleanTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = b.data.Bytes()
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BooleanBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *BooleanBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.init)
b.data.Resize(arrow.BooleanTraits.BytesRequired(n))
b.rawData = b.data.Bytes()
}
}
// NewArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewArray() Interface {
return b.NewBooleanArray()
}
// NewBooleanArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewBooleanArray() (a *Boolean) {
data := b.newData()
a = NewBooleanData(data)
data.Release()
return
}
func (b *BooleanBuilder) newData() *Data {
bytesRequired := arrow.BooleanTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return res
}
var (
_ Builder = (*BooleanBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go
Generated Vendored
-127
View File
@@ -1,127 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A bufferBuilder provides common functionality for populating memory with a sequence of type-specific values.
// Specialized implementations provide type-safe APIs for appending and accessing the memory.
type bufferBuilder struct {
refCount int64
mem memory.Allocator
buffer *memory.Buffer
length int
capacity int
bytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.buffer != nil {
b.buffer.Release()
b.buffer, b.bytes = nil, nil
}
}
}
// Len returns the length of the memory buffer in bytes.
func (b *bufferBuilder) Len() int { return b.length }
// Cap returns the total number of bytes that can be stored without allocating additional memory.
func (b *bufferBuilder) Cap() int { return b.capacity }
// Bytes returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *bufferBuilder) Bytes() []byte { return b.bytes[:b.length] }
func (b *bufferBuilder) resize(elements int) {
if b.buffer == nil {
b.buffer = memory.NewResizableBuffer(b.mem)
}
b.buffer.Resize(elements)
oldCapacity := b.capacity
b.capacity = b.buffer.Cap()
b.bytes = b.buffer.Buf()
if b.capacity > oldCapacity {
memory.Set(b.bytes[oldCapacity:], 0)
}
}
// Advance increases the buffer by length and initializes the skipped bytes to zero.
func (b *bufferBuilder) Advance(length int) {
if b.capacity < b.length+length {
newCapacity := bitutil.NextPowerOf2(b.length + length)
b.resize(newCapacity)
}
b.length += length
}
// Append appends the contents of v to the buffer, resizing it if necessary.
func (b *bufferBuilder) Append(v []byte) {
if b.capacity < b.length+len(v) {
newCapacity := bitutil.NextPowerOf2(b.length + len(v))
b.resize(newCapacity)
}
b.unsafeAppend(v)
}
// Reset returns the buffer to an empty state. Reset releases the memory and sets the length and capacity to zero.
func (b *bufferBuilder) Reset() {
if b.buffer != nil {
b.buffer.Release()
}
b.buffer, b.bytes = nil, nil
b.capacity, b.length = 0, 0
}
// Finish TODO(sgc)
func (b *bufferBuilder) Finish() (buffer *memory.Buffer) {
if b.length > 0 {
b.buffer.ResizeNoShrink(b.length)
}
buffer = b.buffer
b.buffer = nil
b.Reset()
return
}
func (b *bufferBuilder) unsafeAppend(data []byte) {
copy(b.bytes[b.length:], data)
b.length += len(data)
}
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go
Generated Vendored
-30
View File
@@ -1,30 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import "github.com/apache/arrow/go/arrow/memory"
type byteBufferBuilder struct {
bufferBuilder
}
func newByteBufferBuilder(mem memory.Allocator) *byteBufferBuilder {
return &byteBufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
func (b *byteBufferBuilder) Values() []byte { return b.Bytes() }
func (b *byteBufferBuilder) Value(i int) byte { return b.bytes[i] }
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go
Generated Vendored
-58
View File
@@ -1,58 +0,0 @@
// Code generated by array/bufferbuilder_numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
type int32BufferBuilder struct {
bufferBuilder
}
func newInt32BufferBuilder(mem memory.Allocator) *int32BufferBuilder {
return &int32BufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValues(v []int32) { b.Append(arrow.Int32Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *int32BufferBuilder) Values() []int32 { return arrow.Int32Traits.CastFromBytes(b.Bytes()) }
// Value returns the int32 element at the index i. Value will panic if i is negative or ≥ Len.
func (b *int32BufferBuilder) Value(i int) int32 { return b.Values()[i] }
// Len returns the number of int32 elements in the buffer.
func (b *int32BufferBuilder) Len() int { return b.length / arrow.Int32SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValue(v int32) {
if b.capacity < b.length+arrow.Int32SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.Int32SizeBytes)
b.resize(newCapacity)
}
arrow.Int32Traits.PutValue(b.bytes[b.length:], v)
b.length += arrow.Int32SizeBytes
}
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl
Generated Vendored
-61
View File
@@ -1,61 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
{{range .In}}
{{$TypeNamePrefix := .name}}
{{if .Opt.BufferBuilder}}
type {{$TypeNamePrefix}}BufferBuilder struct {
bufferBuilder
}
func new{{.Name}}BufferBuilder(mem memory.Allocator) *{{$TypeNamePrefix}}BufferBuilder {
return &{{$TypeNamePrefix}}BufferBuilder{bufferBuilder:bufferBuilder{refCount: 1, mem:mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValues(v []{{.Type}}) { b.Append(arrow.{{.Name}}Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *{{$TypeNamePrefix}}BufferBuilder) Values() []{{.Type}} { return arrow.{{.Name}}Traits.CastFromBytes(b.Bytes()) }
// Value returns the {{.Type}} element at the index i. Value will panic if i is negative or ≥ Len.
func (b *{{$TypeNamePrefix}}BufferBuilder) Value(i int) {{.Type}} { return b.Values()[i] }
// Len returns the number of {{.Type}} elements in the buffer.
func (b *{{$TypeNamePrefix}}BufferBuilder) Len() int { return b.length/arrow.{{.Name}}SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValue(v {{.Type}}) {
if b.capacity < b.length+arrow.{{.Name}}SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.{{.Name}}SizeBytes)
b.resize(newCapacity)
}
arrow.{{.Name}}Traits.PutValue(b.bytes[b.length:], v)
b.length+=arrow.{{.Name}}SizeBytes
}
{{end}}
{{end}}
vendor/github.com/apache/arrow/go/arrow/array/builder.go
Generated Vendored
-273
View File
@@ -1,273 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
const (
minBuilderCapacity = 1 << 5
)
// Builder provides an interface to build arrow arrays.
type Builder interface {
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
// Release decreases the reference count by 1.
Release()
// Len returns the number of elements in the array builder.
Len() int
// Cap returns the total number of elements that can be stored
// without allocating additional memory.
Cap() int
// NullN returns the number of null values in the array builder.
NullN() int
// AppendNull adds a new null value to the array being built.
AppendNull()
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
Reserve(n int)
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
Resize(n int)
// NewArray creates a new array from the memory buffers used
// by the builder and resets the Builder so it can be used to build
// a new array.
NewArray() Interface
init(capacity int)
resize(newBits int, init func(int))
}
// builder provides common functionality for managing the validity bitmap (nulls) when building arrays.
type builder struct {
refCount int64
mem memory.Allocator
nullBitmap *memory.Buffer
nulls int
length int
capacity int
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *builder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Len returns the number of elements in the array builder.
func (b *builder) Len() int { return b.length }
// Cap returns the total number of elements that can be stored without allocating additional memory.
func (b *builder) Cap() int { return b.capacity }
// NullN returns the number of null values in the array builder.
func (b *builder) NullN() int { return b.nulls }
func (b *builder) init(capacity int) {
toAlloc := bitutil.CeilByte(capacity) / 8
b.nullBitmap = memory.NewResizableBuffer(b.mem)
b.nullBitmap.Resize(toAlloc)
b.capacity = capacity
memory.Set(b.nullBitmap.Buf(), 0)
}
func (b *builder) reset() {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
b.nulls = 0
b.length = 0
b.capacity = 0
}
func (b *builder) resize(newBits int, init func(int)) {
if b.nullBitmap == nil {
init(newBits)
return
}
newBytesN := bitutil.CeilByte(newBits) / 8
oldBytesN := b.nullBitmap.Len()
b.nullBitmap.Resize(newBytesN)
b.capacity = newBits
if oldBytesN < newBytesN {
// TODO(sgc): necessary?
memory.Set(b.nullBitmap.Buf()[oldBytesN:], 0)
}
if newBits < b.length {
b.length = newBits
b.nulls = newBits - bitutil.CountSetBits(b.nullBitmap.Buf(), 0, newBits)
}
}
func (b *builder) reserve(elements int, resize func(int)) {
if b.length+elements > b.capacity {
newCap := bitutil.NextPowerOf2(b.length + elements)
resize(newCap)
}
}
// unsafeAppendBoolsToBitmap appends the contents of valid to the validity bitmap.
// As an optimization, if the valid slice is empty, the next length bits will be set to valid (not null).
func (b *builder) unsafeAppendBoolsToBitmap(valid []bool, length int) {
if len(valid) == 0 {
b.unsafeSetValid(length)
return
}
byteOffset := b.length / 8
bitOffset := byte(b.length % 8)
nullBitmap := b.nullBitmap.Bytes()
bitSet := nullBitmap[byteOffset]
for _, v := range valid {
if bitOffset == 8 {
bitOffset = 0
nullBitmap[byteOffset] = bitSet
byteOffset++
bitSet = nullBitmap[byteOffset]
}
if v {
bitSet |= bitutil.BitMask[bitOffset]
} else {
bitSet &= bitutil.FlippedBitMask[bitOffset]
b.nulls++
}
bitOffset++
}
if bitOffset != 0 {
nullBitmap[byteOffset] = bitSet
}
b.length += len(valid)
}
// unsafeSetValid sets the next length bits to valid in the validity bitmap.
func (b *builder) unsafeSetValid(length int) {
padToByte := min(8-(b.length%8), length)
if padToByte == 8 {
padToByte = 0
}
bits := b.nullBitmap.Bytes()
for i := b.length; i < b.length+padToByte; i++ {
bitutil.SetBit(bits, i)
}
start := (b.length + padToByte) / 8
fastLength := (length - padToByte) / 8
memory.Set(bits[start:start+fastLength], 0xff)
newLength := b.length + length
// trailing bytes
for i := b.length + padToByte + (fastLength * 8); i < newLength; i++ {
bitutil.SetBit(bits, i)
}
b.length = newLength
}
func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func NewBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
// FIXME(sbinet): use a type switch on dtype instead?
switch dtype.ID() {
case arrow.NULL:
case arrow.BOOL:
return NewBooleanBuilder(mem)
case arrow.UINT8:
return NewUint8Builder(mem)
case arrow.INT8:
return NewInt8Builder(mem)
case arrow.UINT16:
return NewUint16Builder(mem)
case arrow.INT16:
return NewInt16Builder(mem)
case arrow.UINT32:
return NewUint32Builder(mem)
case arrow.INT32:
return NewInt32Builder(mem)
case arrow.UINT64:
return NewUint64Builder(mem)
case arrow.INT64:
return NewInt64Builder(mem)
case arrow.FLOAT16:
return NewFloat16Builder(mem)
case arrow.FLOAT32:
return NewFloat32Builder(mem)
case arrow.FLOAT64:
return NewFloat64Builder(mem)
case arrow.STRING:
return NewStringBuilder(mem)
case arrow.BINARY:
return NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
case arrow.FIXED_SIZE_BINARY:
typ := dtype.(*arrow.FixedSizeBinaryType)
return NewFixedSizeBinaryBuilder(mem, typ)
case arrow.DATE32:
case arrow.DATE64:
case arrow.TIMESTAMP:
case arrow.TIME32:
typ := dtype.(*arrow.Time32Type)
return NewTime32Builder(mem, typ)
case arrow.TIME64:
typ := dtype.(*arrow.Time64Type)
return NewTime64Builder(mem, typ)
case arrow.INTERVAL:
case arrow.DECIMAL:
case arrow.LIST:
typ := dtype.(*arrow.ListType)
return NewListBuilder(mem, typ.Elem())
case arrow.STRUCT:
typ := dtype.(*arrow.StructType)
return NewStructBuilder(mem, typ)
case arrow.UNION:
case arrow.DICTIONARY:
case arrow.MAP:
case arrow.EXTENSION:
case arrow.FIXED_SIZE_LIST:
typ := dtype.(*arrow.FixedSizeListType)
return NewFixedSizeListBuilder(mem, typ.Len(), typ.Elem())
case arrow.DURATION:
}
panic(fmt.Errorf("arrow/array: unsupported builder for %T", dtype))
}
vendor/github.com/apache/arrow/go/arrow/array/compare.go
Generated Vendored
-474
View File
@@ -1,474 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"math"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/float16"
"golang.org/x/xerrors"
)
// RecordEqual reports whether the two provided records are equal.
func RecordEqual(left, right Record) bool {
switch {
case left.NumCols() != right.NumCols():
return false
case left.NumRows() != right.NumRows():
return false
}
for i := range left.Columns() {
lc := left.Column(i)
rc := right.Column(i)
if !ArrayEqual(lc, rc) {
return false
}
}
return true
}
// RecordApproxEqual reports whether the two provided records are approximately equal.
// For non-floating point columns, it is equivalent to RecordEqual.
func RecordApproxEqual(left, right Record, opts ...EqualOption) bool {
switch {
case left.NumCols() != right.NumCols():
return false
case left.NumRows() != right.NumRows():
return false
}
opt := newEqualOption(opts...)
for i := range left.Columns() {
lc := left.Column(i)
rc := right.Column(i)
if !arrayApproxEqual(lc, rc, opt) {
return false
}
}
return true
}
// ArrayEqual reports whether the two provided arrays are equal.
func ArrayEqual(left, right Interface) bool {
switch {
case !baseArrayEqual(left, right):
return false
case left.Len() == 0:
return true
case left.NullN() == left.Len():
return true
}
// at this point, we know both arrays have same type, same length, same number of nulls
// and nulls at the same place.
// compare the values.
switch l := left.(type) {
case *Null:
return true
case *Boolean:
r := right.(*Boolean)
return arrayEqualBoolean(l, r)
case *FixedSizeBinary:
r := right.(*FixedSizeBinary)
return arrayEqualFixedSizeBinary(l, r)
case *Binary:
r := right.(*Binary)
return arrayEqualBinary(l, r)
case *String:
r := right.(*String)
return arrayEqualString(l, r)
case *Int8:
r := right.(*Int8)
return arrayEqualInt8(l, r)
case *Int16:
r := right.(*Int16)
return arrayEqualInt16(l, r)
case *Int32:
r := right.(*Int32)
return arrayEqualInt32(l, r)
case *Int64:
r := right.(*Int64)
return arrayEqualInt64(l, r)
case *Uint8:
r := right.(*Uint8)
return arrayEqualUint8(l, r)
case *Uint16:
r := right.(*Uint16)
return arrayEqualUint16(l, r)
case *Uint32:
r := right.(*Uint32)
return arrayEqualUint32(l, r)
case *Uint64:
r := right.(*Uint64)
return arrayEqualUint64(l, r)
case *Float16:
r := right.(*Float16)
return arrayEqualFloat16(l, r)
case *Float32:
r := right.(*Float32)
return arrayEqualFloat32(l, r)
case *Float64:
r := right.(*Float64)
return arrayEqualFloat64(l, r)
case *Decimal128:
r := right.(*Decimal128)
return arrayEqualDecimal128(l, r)
case *Date32:
r := right.(*Date32)
return arrayEqualDate32(l, r)
case *Date64:
r := right.(*Date64)
return arrayEqualDate64(l, r)
case *Time32:
r := right.(*Time32)
return arrayEqualTime32(l, r)
case *Time64:
r := right.(*Time64)
return arrayEqualTime64(l, r)
case *Timestamp:
r := right.(*Timestamp)
return arrayEqualTimestamp(l, r)
case *List:
r := right.(*List)
return arrayEqualList(l, r)
case *FixedSizeList:
r := right.(*FixedSizeList)
return arrayEqualFixedSizeList(l, r)
case *Struct:
r := right.(*Struct)
return arrayEqualStruct(l, r)
case *MonthInterval:
r := right.(*MonthInterval)
return arrayEqualMonthInterval(l, r)
case *DayTimeInterval:
r := right.(*DayTimeInterval)
return arrayEqualDayTimeInterval(l, r)
case *Duration:
r := right.(*Duration)
return arrayEqualDuration(l, r)
default:
panic(xerrors.Errorf("arrow/array: unknown array type %T", l))
}
}
// ArraySliceEqual reports whether slices left[lbeg:lend] and right[rbeg:rend] are equal.
func ArraySliceEqual(left Interface, lbeg, lend int64, right Interface, rbeg, rend int64) bool {
l := NewSlice(left, lbeg, lend)
defer l.Release()
r := NewSlice(right, rbeg, rend)
defer r.Release()
return ArrayEqual(l, r)
}
const defaultAbsoluteTolerance = 1e-5
type equalOption struct {
atol float64 // absolute tolerance
nansEq bool // whether NaNs are considered equal.
}
func (eq equalOption) f16(f1, f2 float16.Num) bool {
v1 := float64(f1.Float32())
v2 := float64(f2.Float32())
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func (eq equalOption) f32(f1, f2 float32) bool {
v1 := float64(f1)
v2 := float64(f2)
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func (eq equalOption) f64(v1, v2 float64) bool {
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func newEqualOption(opts ...EqualOption) equalOption {
eq := equalOption{
atol: defaultAbsoluteTolerance,
nansEq: false,
}
for _, opt := range opts {
opt(&eq)
}
return eq
}
// EqualOption is a functional option type used to configure how Records and Arrays are compared.
type EqualOption func(*equalOption)
// WithNaNsEqual configures the comparison functions so that NaNs are considered equal.
func WithNaNsEqual(v bool) EqualOption {
return func(o *equalOption) {
o.nansEq = v
}
}
// WithAbsTolerance configures the comparison functions so that 2 floating point values
// v1 and v2 are considered equal if |v1-v2| <= atol.
func WithAbsTolerance(atol float64) EqualOption {
return func(o *equalOption) {
o.atol = atol
}
}
// ArrayApproxEqual reports whether the two provided arrays are approximately equal.
// For non-floating point arrays, it is equivalent to ArrayEqual.
func ArrayApproxEqual(left, right Interface, opts ...EqualOption) bool {
opt := newEqualOption(opts...)
return arrayApproxEqual(left, right, opt)
}
func arrayApproxEqual(left, right Interface, opt equalOption) bool {
switch {
case !baseArrayEqual(left, right):
return false
case left.Len() == 0:
return true
case left.NullN() == left.Len():
return true
}
// at this point, we know both arrays have same type, same length, same number of nulls
// and nulls at the same place.
// compare the values.
switch l := left.(type) {
case *Null:
return true
case *Boolean:
r := right.(*Boolean)
return arrayEqualBoolean(l, r)
case *FixedSizeBinary:
r := right.(*FixedSizeBinary)
return arrayEqualFixedSizeBinary(l, r)
case *Binary:
r := right.(*Binary)
return arrayEqualBinary(l, r)
case *String:
r := right.(*String)
return arrayEqualString(l, r)
case *Int8:
r := right.(*Int8)
return arrayEqualInt8(l, r)
case *Int16:
r := right.(*Int16)
return arrayEqualInt16(l, r)
case *Int32:
r := right.(*Int32)
return arrayEqualInt32(l, r)
case *Int64:
r := right.(*Int64)
return arrayEqualInt64(l, r)
case *Uint8:
r := right.(*Uint8)
return arrayEqualUint8(l, r)
case *Uint16:
r := right.(*Uint16)
return arrayEqualUint16(l, r)
case *Uint32:
r := right.(*Uint32)
return arrayEqualUint32(l, r)
case *Uint64:
r := right.(*Uint64)
return arrayEqualUint64(l, r)
case *Float16:
r := right.(*Float16)
return arrayApproxEqualFloat16(l, r, opt)
case *Float32:
r := right.(*Float32)
return arrayApproxEqualFloat32(l, r, opt)
case *Float64:
r := right.(*Float64)
return arrayApproxEqualFloat64(l, r, opt)
case *Decimal128:
r := right.(*Decimal128)
return arrayEqualDecimal128(l, r)
case *Date32:
r := right.(*Date32)
return arrayEqualDate32(l, r)
case *Date64:
r := right.(*Date64)
return arrayEqualDate64(l, r)
case *Time32:
r := right.(*Time32)
return arrayEqualTime32(l, r)
case *Time64:
r := right.(*Time64)
return arrayEqualTime64(l, r)
case *Timestamp:
r := right.(*Timestamp)
return arrayEqualTimestamp(l, r)
case *List:
r := right.(*List)
return arrayApproxEqualList(l, r, opt)
case *FixedSizeList:
r := right.(*FixedSizeList)
return arrayApproxEqualFixedSizeList(l, r, opt)
case *Struct:
r := right.(*Struct)
return arrayApproxEqualStruct(l, r, opt)
case *MonthInterval:
r := right.(*MonthInterval)
return arrayEqualMonthInterval(l, r)
case *DayTimeInterval:
r := right.(*DayTimeInterval)
return arrayEqualDayTimeInterval(l, r)
case *Duration:
r := right.(*Duration)
return arrayEqualDuration(l, r)
default:
panic(xerrors.Errorf("arrow/array: unknown array type %T", l))
}
return false
}
func baseArrayEqual(left, right Interface) bool {
switch {
case left.Len() != right.Len():
return false
case left.NullN() != right.NullN():
return false
case !arrow.TypeEqual(left.DataType(), right.DataType()): // We do not check for metadata as in the C++ implementation.
return false
case !validityBitmapEqual(left, right):
return false
}
return true
}
func validityBitmapEqual(left, right Interface) bool {
// TODO(alexandreyc): make it faster by comparing byte slices of the validity bitmap?
n := left.Len()
if n != right.Len() {
return false
}
for i := 0; i < n; i++ {
if left.IsNull(i) != right.IsNull(i) {
return false
}
}
return true
}
func arrayApproxEqualFloat16(left, right *Float16, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f16(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualFloat32(left, right *Float32, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f32(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualFloat64(left, right *Float64, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f64(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualList(left, right *List, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return arrayApproxEqual(l, r, opt)
}()
if !o {
return false
}
}
return true
}
func arrayApproxEqualFixedSizeList(left, right *FixedSizeList, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return arrayApproxEqual(l, r, opt)
}()
if !o {
return false
}
}
return true
}
func arrayApproxEqualStruct(left, right *Struct, opt equalOption) bool {
for i, lf := range left.fields {
rf := right.fields[i]
if !arrayApproxEqual(lf, rf, opt) {
return false
}
}
return true
}
vendor/github.com/apache/arrow/go/arrow/array/data.go
Generated Vendored
-179
View File
@@ -1,179 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// Data represents the memory and metadata of an Arrow array.
type Data struct {
refCount int64
dtype arrow.DataType
nulls int
offset int
length int
buffers []*memory.Buffer // TODO(sgc): should this be an interface?
childData []*Data // TODO(sgc): managed by ListArray, StructArray and UnionArray types
}
// NewData creates a new Data.
func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls, offset int) *Data {
for _, b := range buffers {
if b != nil {
b.Retain()
}
}
for _, child := range childData {
if child != nil {
child.Retain()
}
}
return &Data{
refCount: 1,
dtype: dtype,
nulls: nulls,
length: length,
offset: offset,
buffers: buffers,
childData: childData,
}
}
// Reset sets the Data for re-use.
func (d *Data) Reset(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls, offset int) {
// Retain new buffers before releasing existing buffers in-case they're the same ones to prevent accidental premature
// release.
for _, b := range buffers {
if b != nil {
b.Retain()
}
}
for _, b := range d.buffers {
if b != nil {
b.Release()
}
}
d.buffers = buffers
// Retain new children data before releasing existing children data in-case they're the same ones to prevent accidental
// premature release.
for _, d := range childData {
if d != nil {
d.Retain()
}
}
for _, d := range d.childData {
if d != nil {
d.Release()
}
}
d.childData = childData
d.dtype = dtype
d.length = length
d.nulls = nulls
d.offset = offset
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (d *Data) Retain() {
atomic.AddInt64(&d.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (d *Data) Release() {
debug.Assert(atomic.LoadInt64(&d.refCount) > 0, "too many releases")
if atomic.AddInt64(&d.refCount, -1) == 0 {
for _, b := range d.buffers {
if b != nil {
b.Release()
}
}
for _, b := range d.childData {
b.Release()
}
d.buffers, d.childData = nil, nil
}
}
// DataType returns the DataType of the data.
func (d *Data) DataType() arrow.DataType { return d.dtype }
// NullN returns the number of nulls.
func (d *Data) NullN() int { return d.nulls }
// Len returns the length.
func (d *Data) Len() int { return d.length }
// Offset returns the offset.
func (d *Data) Offset() int { return d.offset }
// Buffers returns the buffers.
func (d *Data) Buffers() []*memory.Buffer { return d.buffers }
// NewSliceData returns a new slice that shares backing data with the input.
// The returned Data slice starts at i and extends j-i elements, such as:
// slice := data[i:j]
// The returned value must be Release'd after use.
//
// NewSliceData panics if the slice is outside the valid range of the input Data.
// NewSliceData panics if j < i.
func NewSliceData(data *Data, i, j int64) *Data {
if j > int64(data.length) || i > j || data.offset+int(i) > data.offset+data.length {
panic("arrow/array: index out of range")
}
for _, b := range data.buffers {
if b != nil {
b.Retain()
}
}
for _, child := range data.childData {
if child != nil {
child.Retain()
}
}
o := &Data{
refCount: 1,
dtype: data.dtype,
nulls: UnknownNullCount,
length: int(j - i),
offset: data.offset + int(i),
buffers: data.buffers,
childData: data.childData,
}
if data.nulls == 0 {
o.nulls = 0
}
return o
}
vendor/github.com/apache/arrow/go/arrow/array/decimal128.go
Generated Vendored
-235
View File
@@ -1,235 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/decimal128"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents an immutable sequence of 128-bit decimal values.
type Decimal128 struct {
array
values []decimal128.Num
}
func NewDecimal128Data(data *Data) *Decimal128 {
a := &Decimal128{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Decimal128) Value(i int) decimal128.Num { return a.values[i] }
func (a *Decimal128) Values() []decimal128.Num { return a.values }
func (a *Decimal128) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Decimal128) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Decimal128Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDecimal128(left, right *Decimal128) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type Decimal128Builder struct {
builder
dtype *arrow.Decimal128Type
data *memory.Buffer
rawData []decimal128.Num
}
func NewDecimal128Builder(mem memory.Allocator, dtype *arrow.Decimal128Type) *Decimal128Builder {
return &Decimal128Builder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *Decimal128Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *Decimal128Builder) Append(v decimal128.Num) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *Decimal128Builder) UnsafeAppend(v decimal128.Num) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *Decimal128Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *Decimal128Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *Decimal128Builder) AppendValues(v []decimal128.Num, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
if len(v) > 0 {
arrow.Decimal128Traits.Copy(b.rawData[b.length:], v)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *Decimal128Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.Decimal128Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *Decimal128Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *Decimal128Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.Decimal128Traits.BytesRequired(n))
b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder
// so it can be used to build a new array.
func (b *Decimal128Builder) NewArray() Interface {
return b.NewDecimal128Array()
}
// NewDecimal128Array creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder
// so it can be used to build a new array.
func (b *Decimal128Builder) NewDecimal128Array() (a *Decimal128) {
data := b.newData()
a = NewDecimal128Data(data)
data.Release()
return
}
func (b *Decimal128Builder) newData() (data *Data) {
bytesRequired := arrow.Decimal128Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
var (
_ Interface = (*Decimal128)(nil)
_ Builder = (*Decimal128Builder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/doc.go
Generated Vendored
-20
View File
@@ -1,20 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package array provides implementations of various Arrow array types.
*/
package array
vendor/github.com/apache/arrow/go/arrow/array/fixed_size_list.go
Generated Vendored
-240
View File
@@ -1,240 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// FixedSizeList represents an immutable sequence of N array values.
type FixedSizeList struct {
array
n int32
values Interface
}
// NewFixedSizeListData returns a new List array value, from data.
func NewFixedSizeListData(data *Data) *FixedSizeList {
a := &FixedSizeList{}
a.refCount = 1
a.setData(data)
return a
}
func (a *FixedSizeList) ListValues() Interface { return a.values }
func (a *FixedSizeList) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
if !a.IsValid(i) {
o.WriteString("(null)")
continue
}
sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
o.WriteString("]")
return o.String()
}
func (a *FixedSizeList) newListValue(i int) Interface {
n := int64(a.n)
off := int64(a.array.data.offset)
beg := (off + int64(i)) * n
end := (off + int64(i+1)) * n
sli := NewSlice(a.values, beg, end)
return sli
}
func (a *FixedSizeList) setData(data *Data) {
a.array.setData(data)
a.n = a.DataType().(*arrow.FixedSizeListType).Len()
a.values = MakeFromData(data.childData[0])
}
func arrayEqualFixedSizeList(left, right *FixedSizeList) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return ArrayEqual(l, r)
}()
if !o {
return false
}
}
return true
}
// Len returns the number of elements in the array.
func (a *FixedSizeList) Len() int { return a.array.Len() }
func (a *FixedSizeList) Retain() {
a.array.Retain()
a.values.Retain()
}
func (a *FixedSizeList) Release() {
a.array.Release()
a.values.Release()
}
type FixedSizeListBuilder struct {
builder
etype arrow.DataType // data type of the list's elements.
n int32 // number of elements in the fixed-size list.
values Builder // value builder for the list's elements.
}
// NewFixedSizeListBuilder returns a builder, using the provided memory allocator.
// The created list builder will create a list whose elements will be of type etype.
func NewFixedSizeListBuilder(mem memory.Allocator, n int32, etype arrow.DataType) *FixedSizeListBuilder {
return &FixedSizeListBuilder{
builder: builder{refCount: 1, mem: mem},
etype: etype,
n: n,
values: NewBuilder(mem, etype),
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *FixedSizeListBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *FixedSizeListBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
}
func (b *FixedSizeListBuilder) AppendNull() {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(false)
}
func (b *FixedSizeListBuilder) AppendValues(valid []bool) {
b.Reserve(len(valid))
b.builder.unsafeAppendBoolsToBitmap(valid, len(valid))
}
func (b *FixedSizeListBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *FixedSizeListBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *FixedSizeListBuilder) init(capacity int) {
b.builder.init(capacity)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *FixedSizeListBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *FixedSizeListBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
}
}
func (b *FixedSizeListBuilder) ValueBuilder() Builder {
return b.values
}
// NewArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder
// so it can be used to build a new array.
func (b *FixedSizeListBuilder) NewArray() Interface {
return b.NewListArray()
}
// NewListArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder
// so it can be used to build a new array.
func (b *FixedSizeListBuilder) NewListArray() (a *FixedSizeList) {
data := b.newData()
a = NewFixedSizeListData(data)
data.Release()
return
}
func (b *FixedSizeListBuilder) newData() (data *Data) {
values := b.values.NewArray()
defer values.Release()
data = NewData(
arrow.FixedSizeListOf(b.n, b.etype), b.length,
[]*memory.Buffer{b.nullBitmap},
[]*Data{values.Data()},
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*FixedSizeList)(nil)
_ Builder = (*FixedSizeListBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binary.go
Generated Vendored
-95
View File
@@ -1,95 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of fixed-length binary strings.
type FixedSizeBinary struct {
array
valueBytes []byte
bytewidth int32
}
// NewFixedSizeBinaryData constructs a new fixed-size binary array from data.
func NewFixedSizeBinaryData(data *Data) *FixedSizeBinary {
a := &FixedSizeBinary{bytewidth: int32(data.DataType().(arrow.FixedWidthDataType).BitWidth() / 8)}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the fixed-size slice at index i. This value should not be mutated.
func (a *FixedSizeBinary) Value(i int) []byte {
i += a.array.data.offset
var (
bw = int(a.bytewidth)
beg = i * bw
end = (i + 1) * bw
)
return a.valueBytes[beg:end]
}
func (a *FixedSizeBinary) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *FixedSizeBinary) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.valueBytes = vals.Bytes()
}
}
func arrayEqualFixedSizeBinary(left, right *FixedSizeBinary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
return false
}
}
return true
}
var (
_ Interface = (*FixedSizeBinary)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binarybuilder.go
Generated Vendored
-154
View File
@@ -1,154 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods.
type FixedSizeBinaryBuilder struct {
builder
dtype *arrow.FixedSizeBinaryType
values *byteBufferBuilder
}
func NewFixedSizeBinaryBuilder(mem memory.Allocator, dtype *arrow.FixedSizeBinaryType) *FixedSizeBinaryBuilder {
b := &FixedSizeBinaryBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
values: newByteBufferBuilder(mem),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *FixedSizeBinaryBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *FixedSizeBinaryBuilder) Append(v []byte) {
if len(v) != b.dtype.ByteWidth {
// TODO(alexandre): should we return an error instead?
panic("len(v) != b.dtype.ByteWidth")
}
b.Reserve(1)
b.values.Append(v)
b.UnsafeAppendBoolToBitmap(true)
}
func (b *FixedSizeBinaryBuilder) AppendNull() {
b.Reserve(1)
b.values.Advance(b.dtype.ByteWidth)
b.UnsafeAppendBoolToBitmap(false)
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *FixedSizeBinaryBuilder) AppendValues(v [][]byte, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
switch len(vv) {
case 0:
b.values.Advance(b.dtype.ByteWidth)
case b.dtype.ByteWidth:
b.values.Append(vv)
default:
panic(fmt.Errorf("array: invalid binary length (got=%d, want=%d)", len(vv), b.dtype.ByteWidth))
}
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *FixedSizeBinaryBuilder) init(capacity int) {
b.builder.init(capacity)
b.values.resize(capacity * b.dtype.ByteWidth)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *FixedSizeBinaryBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *FixedSizeBinaryBuilder) Resize(n int) {
b.builder.resize(n, b.init)
}
// NewArray creates a FixedSizeBinary array from the memory buffers used by the
// builder and resets the FixedSizeBinaryBuilder so it can be used to build a new array.
func (b *FixedSizeBinaryBuilder) NewArray() Interface {
return b.NewFixedSizeBinaryArray()
}
// NewFixedSizeBinaryArray creates a FixedSizeBinary array from the memory buffers used by the builder and resets the FixedSizeBinaryBuilder
// so it can be used to build a new array.
func (b *FixedSizeBinaryBuilder) NewFixedSizeBinaryArray() (a *FixedSizeBinary) {
data := b.newData()
a = NewFixedSizeBinaryData(data)
data.Release()
return
}
func (b *FixedSizeBinaryBuilder) newData() (data *Data) {
values := b.values.Finish()
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, values}, nil, b.nulls, 0)
if values != nil {
values.Release()
}
b.builder.reset()
return
}
var (
_ Builder = (*FixedSizeBinaryBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/float16.go
Generated Vendored
-87
View File
@@ -1,87 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/float16"
)
// A type which represents an immutable sequence of Float16 values.
type Float16 struct {
array
values []float16.Num
}
func NewFloat16Data(data *Data) *Float16 {
a := &Float16{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Float16) Value(i int) float16.Num { return a.values[i] }
func (a *Float16) Values() []float16.Num { return a.values }
func (a *Float16) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.values[i].Float32())
}
}
o.WriteString("]")
return o.String()
}
func (a *Float16) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Float16Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualFloat16(left, right *Float16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
var (
_ Interface = (*Float16)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/float16_builder.go
Generated Vendored
-165
View File
@@ -1,165 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/float16"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
type Float16Builder struct {
builder
data *memory.Buffer
rawData []float16.Num
}
func NewFloat16Builder(mem memory.Allocator) *Float16Builder {
return &Float16Builder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *Float16Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *Float16Builder) Append(v float16.Num) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *Float16Builder) UnsafeAppend(v float16.Num) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *Float16Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *Float16Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *Float16Builder) AppendValues(v []float16.Num, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
if len(v) > 0 {
arrow.Float16Traits.Copy(b.rawData[b.length:], v)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *Float16Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.Uint16Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *Float16Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *Float16Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.Float16Traits.BytesRequired(n))
b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder
// so it can be used to build a new array.
func (b *Float16Builder) NewArray() Interface {
return b.NewFloat16Array()
}
// NewFloat16Array creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder
// so it can be used to build a new array.
func (b *Float16Builder) NewFloat16Array() (a *Float16) {
data := b.newData()
a = NewFloat16Data(data)
data.Release()
return
}
func (b *Float16Builder) newData() (data *Data) {
bytesRequired := arrow.Float16Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.Float16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
vendor/github.com/apache/arrow/go/arrow/array/interval.go
Generated Vendored
-434
View File
@@ -1,434 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
"golang.org/x/xerrors"
)
func NewIntervalData(data *Data) Interface {
switch data.dtype.(type) {
case *arrow.MonthIntervalType:
return NewMonthIntervalData(data)
case *arrow.DayTimeIntervalType:
return NewDayTimeIntervalData(data)
default:
panic(xerrors.Errorf("arrow/array: unknown interval data type %T", data.dtype))
}
}
// A type which represents an immutable sequence of arrow.MonthInterval values.
type MonthInterval struct {
array
values []arrow.MonthInterval
}
func NewMonthIntervalData(data *Data) *MonthInterval {
a := &MonthInterval{}
a.refCount = 1
a.setData(data)
return a
}
func (a *MonthInterval) Value(i int) arrow.MonthInterval { return a.values[i] }
func (a *MonthInterval) MonthIntervalValues() []arrow.MonthInterval { return a.values }
func (a *MonthInterval) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *MonthInterval) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.MonthIntervalTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualMonthInterval(left, right *MonthInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type MonthIntervalBuilder struct {
builder
data *memory.Buffer
rawData []arrow.MonthInterval
}
func NewMonthIntervalBuilder(mem memory.Allocator) *MonthIntervalBuilder {
return &MonthIntervalBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *MonthIntervalBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *MonthIntervalBuilder) Append(v arrow.MonthInterval) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *MonthIntervalBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *MonthIntervalBuilder) UnsafeAppend(v arrow.MonthInterval) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *MonthIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *MonthIntervalBuilder) AppendValues(v []arrow.MonthInterval, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.MonthIntervalTraits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *MonthIntervalBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.MonthIntervalTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *MonthIntervalBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *MonthIntervalBuilder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.MonthIntervalTraits.BytesRequired(n))
b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder
// so it can be used to build a new array.
func (b *MonthIntervalBuilder) NewArray() Interface {
return b.NewMonthIntervalArray()
}
// NewMonthIntervalArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder
// so it can be used to build a new array.
func (b *MonthIntervalBuilder) NewMonthIntervalArray() (a *MonthInterval) {
data := b.newData()
a = NewMonthIntervalData(data)
data.Release()
return
}
func (b *MonthIntervalBuilder) newData() (data *Data) {
bytesRequired := arrow.MonthIntervalTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.MonthInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
// A type which represents an immutable sequence of arrow.DayTimeInterval values.
type DayTimeInterval struct {
array
values []arrow.DayTimeInterval
}
func NewDayTimeIntervalData(data *Data) *DayTimeInterval {
a := &DayTimeInterval{}
a.refCount = 1
a.setData(data)
return a
}
func (a *DayTimeInterval) Value(i int) arrow.DayTimeInterval { return a.values[i] }
func (a *DayTimeInterval) DayTimeIntervalValues() []arrow.DayTimeInterval { return a.values }
func (a *DayTimeInterval) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *DayTimeInterval) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.DayTimeIntervalTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDayTimeInterval(left, right *DayTimeInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type DayTimeIntervalBuilder struct {
builder
data *memory.Buffer
rawData []arrow.DayTimeInterval
}
func NewDayTimeIntervalBuilder(mem memory.Allocator) *DayTimeIntervalBuilder {
return &DayTimeIntervalBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *DayTimeIntervalBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *DayTimeIntervalBuilder) Append(v arrow.DayTimeInterval) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *DayTimeIntervalBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *DayTimeIntervalBuilder) UnsafeAppend(v arrow.DayTimeInterval) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *DayTimeIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *DayTimeIntervalBuilder) AppendValues(v []arrow.DayTimeInterval, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.DayTimeIntervalTraits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *DayTimeIntervalBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.DayTimeIntervalTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *DayTimeIntervalBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *DayTimeIntervalBuilder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.DayTimeIntervalTraits.BytesRequired(n))
b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder
// so it can be used to build a new array.
func (b *DayTimeIntervalBuilder) NewArray() Interface {
return b.NewDayTimeIntervalArray()
}
// NewDayTimeIntervalArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder
// so it can be used to build a new array.
func (b *DayTimeIntervalBuilder) NewDayTimeIntervalArray() (a *DayTimeInterval) {
data := b.newData()
a = NewDayTimeIntervalData(data)
data.Release()
return
}
func (b *DayTimeIntervalBuilder) newData() (data *Data) {
bytesRequired := arrow.DayTimeIntervalTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.DayTimeInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
var (
_ Interface = (*MonthInterval)(nil)
_ Interface = (*DayTimeInterval)(nil)
_ Builder = (*MonthIntervalBuilder)(nil)
_ Builder = (*DayTimeIntervalBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/list.go
Generated Vendored
-269
View File
@@ -1,269 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// List represents an immutable sequence of array values.
type List struct {
array
values Interface
offsets []int32
}
// NewListData returns a new List array value, from data.
func NewListData(data *Data) *List {
a := &List{}
a.refCount = 1
a.setData(data)
return a
}
func (a *List) ListValues() Interface { return a.values }
func (a *List) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
if !a.IsValid(i) {
o.WriteString("(null)")
continue
}
sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
o.WriteString("]")
return o.String()
}
func (a *List) newListValue(i int) Interface {
j := i + a.array.data.offset
beg := int64(a.offsets[j])
end := int64(a.offsets[j+1])
return NewSlice(a.values, beg, end)
}
func (a *List) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.offsets = arrow.Int32Traits.CastFromBytes(vals.Bytes())
}
a.values = MakeFromData(data.childData[0])
}
func arrayEqualList(left, right *List) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return ArrayEqual(l, r)
}()
if !o {
return false
}
}
return true
}
// Len returns the number of elements in the array.
func (a *List) Len() int { return a.array.Len() }
func (a *List) Offsets() []int32 { return a.offsets }
func (a *List) Retain() {
a.array.Retain()
a.values.Retain()
}
func (a *List) Release() {
a.array.Release()
a.values.Release()
}
type ListBuilder struct {
builder
etype arrow.DataType // data type of the list's elements.
values Builder // value builder for the list's elements.
offsets *Int32Builder
}
// NewListBuilder returns a builder, using the provided memory allocator.
// The created list builder will create a list whose elements will be of type etype.
func NewListBuilder(mem memory.Allocator, etype arrow.DataType) *ListBuilder {
return &ListBuilder{
builder: builder{refCount: 1, mem: mem},
etype: etype,
values: NewBuilder(mem, etype),
offsets: NewInt32Builder(mem),
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *ListBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
b.values.Release()
b.offsets.Release()
}
func (b *ListBuilder) appendNextOffset() {
b.offsets.Append(int32(b.values.Len()))
}
func (b *ListBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
b.appendNextOffset()
}
func (b *ListBuilder) AppendNull() {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(false)
b.appendNextOffset()
}
func (b *ListBuilder) AppendValues(offsets []int32, valid []bool) {
b.Reserve(len(valid))
b.offsets.AppendValues(offsets, nil)
b.builder.unsafeAppendBoolsToBitmap(valid, len(valid))
}
func (b *ListBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *ListBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *ListBuilder) init(capacity int) {
b.builder.init(capacity)
b.offsets.init(capacity + 1)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *ListBuilder) Reserve(n int) {
b.builder.reserve(n, b.resizeHelper)
b.offsets.Reserve(n)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *ListBuilder) Resize(n int) {
b.resizeHelper(n)
b.offsets.Resize(n)
}
func (b *ListBuilder) resizeHelper(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
}
}
func (b *ListBuilder) ValueBuilder() Builder {
return b.values
}
// NewArray creates a List array from the memory buffers used by the builder and resets the ListBuilder
// so it can be used to build a new array.
func (b *ListBuilder) NewArray() Interface {
return b.NewListArray()
}
// NewListArray creates a List array from the memory buffers used by the builder and resets the ListBuilder
// so it can be used to build a new array.
func (b *ListBuilder) NewListArray() (a *List) {
if b.offsets.Len() != b.length+1 {
b.appendNextOffset()
}
data := b.newData()
a = NewListData(data)
data.Release()
return
}
func (b *ListBuilder) newData() (data *Data) {
values := b.values.NewArray()
defer values.Release()
var offsets *memory.Buffer
if b.offsets != nil {
arr := b.offsets.NewInt32Array()
defer arr.Release()
offsets = arr.Data().buffers[1]
}
data = NewData(
arrow.ListOf(b.etype), b.length,
[]*memory.Buffer{
b.nullBitmap,
offsets,
},
[]*Data{values.Data()},
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*List)(nil)
_ Builder = (*ListBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/null.go
Generated Vendored
-140
View File
@@ -1,140 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// Null represents an immutable, degenerate array with no physical storage.
type Null struct {
array
}
// NewNull returns a new Null array value of size n.
func NewNull(n int) *Null {
a := &Null{}
a.refCount = 1
data := NewData(
arrow.Null, n,
[]*memory.Buffer{nil},
nil,
n,
0,
)
a.setData(data)
data.Release()
return a
}
// NewNullData returns a new Null array value, from data.
func NewNullData(data *Data) *Null {
a := &Null{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Null) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
o.WriteString("(null)")
}
o.WriteString("]")
return o.String()
}
func (a *Null) setData(data *Data) {
a.array.setData(data)
a.array.nullBitmapBytes = nil
a.array.data.nulls = a.array.data.length
}
type NullBuilder struct {
builder
}
// NewNullBuilder returns a builder, using the provided memory allocator.
func NewNullBuilder(mem memory.Allocator) *NullBuilder {
return &NullBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *NullBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
}
func (b *NullBuilder) AppendNull() {
b.builder.length++
b.builder.nulls++
}
func (*NullBuilder) Reserve(size int) {}
func (*NullBuilder) Resize(size int) {}
func (*NullBuilder) init(cap int) {}
func (*NullBuilder) resize(newBits int, init func(int)) {}
// NewArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder
// so it can be used to build a new array.
func (b *NullBuilder) NewArray() Interface {
return b.NewNullArray()
}
// NewNullArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder
// so it can be used to build a new array.
func (b *NullBuilder) NewNullArray() (a *Null) {
data := b.newData()
a = NewNullData(data)
data.Release()
return
}
func (b *NullBuilder) newData() (data *Data) {
data = NewData(
arrow.Null, b.length,
[]*memory.Buffer{nil},
nil,
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*Null)(nil)
_ Builder = (*NullBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go
Generated Vendored
-1098
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go.tmpl
Generated Vendored
-95
View File
@@ -1,95 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
)
{{range .In}}
// A type which represents an immutable sequence of {{or .QualifiedType .Type}} values.
type {{.Name}} struct {
array
values []{{or .QualifiedType .Type}}
}
// New{{.Name}}Data creates a new {{.Name}}.
func New{{.Name}}Data(data *Data) *{{.Name}} {
a := &{{.Name}}{}
a.refCount = 1
a.setData(data)
return a
}
// Reset resets the array for re-use.
func (a *{{.Name}}) Reset(data *Data) {
a.setData(data)
}
// Value returns the value at the specified index.
func (a *{{.Name}}) Value(i int) {{or .QualifiedType .Type}} { return a.values[i] }
// Values returns the values.
func (a *{{.Name}}) {{.Name}}Values() []{{or .QualifiedType .Type}} { return a.values }
// String returns a string representation of the array.
func (a *{{.Name}}) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *{{.Name}}) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.{{.Name}}Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqual{{.Name}}(left, right *{{.Name}}) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
{{end}}
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go
Generated Vendored
-2227
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go.tmpl
Generated Vendored
-182
View File
@@ -1,182 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
{{range .In}}
type {{.Name}}Builder struct {
builder
{{if .Opt.Parametric -}}
dtype *arrow.{{.Name}}Type
{{end -}}
data *memory.Buffer
rawData []{{or .QualifiedType .Type}}
}
{{if .Opt.Parametric}}
func New{{.Name}}Builder(mem memory.Allocator, dtype *arrow.{{.Name}}Type) *{{.Name}}Builder {
return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}, dtype: dtype}
}
{{else}}
func New{{.Name}}Builder(mem memory.Allocator) *{{.Name}}Builder {
return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}}
}
{{end}}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *{{.Name}}Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *{{.Name}}Builder) Append(v {{or .QualifiedType .Type}}) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *{{.Name}}Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *{{.Name}}Builder) UnsafeAppend(v {{or .QualifiedType .Type}}) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *{{.Name}}Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *{{.Name}}Builder) AppendValues(v []{{or .QualifiedType .Type}}, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.{{.Name}}Traits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *{{.Name}}Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.{{.Name}}Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *{{.Name}}Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *{{.Name}}Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.{{.Name}}Traits.BytesRequired(n))
b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder
// so it can be used to build a new array.
func (b *{{.Name}}Builder) NewArray() Interface {
return b.New{{.Name}}Array()
}
// New{{.Name}}Array creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder
// so it can be used to build a new array.
func (b *{{.Name}}Builder) New{{.Name}}Array() (a *{{.Name}}) {
data := b.newData()
a = New{{.Name}}Data(data)
data.Release()
return
}
func (b *{{.Name}}Builder) newData() (data *Data) {
bytesRequired := arrow.{{.Name}}Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
{{if .Opt.Parametric -}}
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
{{else -}}
data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
{{end -}}
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
{{end}}
var (
{{- range .In}}
_ Builder = (*{{.Name}}Builder)(nil)
{{- end}}
)
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen_test.go.tmpl
Generated Vendored
-216
View File
@@ -1,216 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array_test
import (
"testing"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/stretchr/testify/assert"
)
{{range .In}}
func TestNew{{.Name}}Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
ab.Retain()
ab.Release()
ab.Append(1)
ab.Append(2)
ab.Append(3)
ab.AppendNull()
ab.Append(5)
ab.Append(6)
ab.AppendNull()
ab.Append(8)
ab.Append(9)
ab.Append(10)
// check state of builder before New{{.Name}}Array
assert.Equal(t, 10, ab.Len(), "unexpected Len()")
assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
a := ab.New{{.Name}}Array()
// check state of builder after New{{.Name}}Array
assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), New{{.Name}}Array did not reset state")
assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), New{{.Name}}Array did not reset state")
assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), New{{.Name}}Array did not reset state")
// check state of array
assert.Equal(t, 2, a.NullN(), "unexpected null count")
assert.Equal(t, []{{or .QualifiedType .Type}}{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.{{.Name}}Values(), "unexpected {{.Name}}Values")
assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity
assert.Len(t, a.{{.Name}}Values(), 10, "unexpected length of {{.Name}}Values")
a.Release()
ab.Append(7)
ab.Append(8)
a = ab.New{{.Name}}Array()
assert.Equal(t, 0, a.NullN())
assert.Equal(t, []{{or .QualifiedType .Type}}{7, 8}, a.{{.Name}}Values())
assert.Len(t, a.{{.Name}}Values(), 2)
a.Release()
var (
want = []{{or .QualifiedType .Type}}{1, 2, 3, 4}
valids = []bool{true, true, false, true}
)
ab.AppendValues(want, valids)
a = ab.New{{.Name}}Array()
sub := array.MakeFromData(a.Data())
defer sub.Release()
if got, want := sub.DataType().ID(), a.DataType().ID(); got != want {
t.Fatalf("invalid type: got=%q, want=%q", got, want)
}
if _, ok := sub.(*array.{{.Name}}); !ok {
t.Fatalf("could not type-assert to array.{{.Name}}")
}
if got, want := a.String(), `[1 2 (null) 4]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
slice := array.NewSliceData(a.Data(), 2, 4)
defer slice.Release()
sub1 := array.MakeFromData(slice)
defer sub1.Release()
v, ok := sub1.(*array.{{.Name}})
if !ok {
t.Fatalf("could not type-assert to array.{{.Name}}")
}
if got, want := v.String(), `[(null) 4]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
a.Release()
}
func Test{{.Name}}Builder_AppendValues(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3}
ab.AppendValues(exp, nil)
a := ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
}
func Test{{.Name}}Builder_Empty(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3}
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
a := ab.New{{.Name}}Array()
assert.Zero(t, a.Len())
a.Release()
ab.AppendValues(nil, nil)
a = ab.New{{.Name}}Array()
assert.Zero(t, a.Len())
a.Release()
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
ab.AppendValues(exp, nil)
a = ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
ab.AppendValues(exp, nil)
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
a = ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
}
func Test{{.Name}}Builder_Resize(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
assert.Equal(t, 0, ab.Cap())
assert.Equal(t, 0, ab.Len())
ab.Reserve(63)
assert.Equal(t, 64, ab.Cap())
assert.Equal(t, 0, ab.Len())
for i := 0; i < 63; i++ {
ab.Append(0)
}
assert.Equal(t, 64, ab.Cap())
assert.Equal(t, 63, ab.Len())
ab.Resize(5)
assert.Equal(t, 5, ab.Len())
ab.Resize(32)
assert.Equal(t, 5, ab.Len())
}
{{end}}
vendor/github.com/apache/arrow/go/arrow/array/record.go
Generated Vendored
-345
View File
@@ -1,345 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// RecordReader reads a stream of records.
type RecordReader interface {
Retain()
Release()
Schema() *arrow.Schema
Next() bool
Record() Record
}
// simpleRecords is a simple iterator over a collection of records.
type simpleRecords struct {
refCount int64
schema *arrow.Schema
recs []Record
cur Record
}
// NewRecordReader returns a simple iterator over the given slice of records.
func NewRecordReader(schema *arrow.Schema, recs []Record) (*simpleRecords, error) {
rs := &simpleRecords{
refCount: 1,
schema: schema,
recs: recs,
cur: nil,
}
for _, rec := range rs.recs {
rec.Retain()
}
for _, rec := range recs {
if !rec.Schema().Equal(rs.schema) {
rs.Release()
return nil, fmt.Errorf("arrow/array: mismatch schema")
}
}
return rs, nil
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (rs *simpleRecords) Retain() {
atomic.AddInt64(&rs.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (rs *simpleRecords) Release() {
debug.Assert(atomic.LoadInt64(&rs.refCount) > 0, "too many releases")
if atomic.AddInt64(&rs.refCount, -1) == 0 {
if rs.cur != nil {
rs.cur.Release()
}
for _, rec := range rs.recs {
rec.Release()
}
rs.recs = nil
}
}
func (rs *simpleRecords) Schema() *arrow.Schema { return rs.schema }
func (rs *simpleRecords) Record() Record { return rs.cur }
func (rs *simpleRecords) Next() bool {
if len(rs.recs) == 0 {
return false
}
if rs.cur != nil {
rs.cur.Release()
}
rs.cur = rs.recs[0]
rs.recs = rs.recs[1:]
return true
}
// Record is a collection of equal-length arrays
// matching a particular Schema.
type Record interface {
Release()
Retain()
Schema() *arrow.Schema
NumRows() int64
NumCols() int64
Columns() []Interface
Column(i int) Interface
ColumnName(i int) string
// NewSlice constructs a zero-copy slice of the record with the indicated
// indices i and j, corresponding to array[i:j].
// The returned record must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the record array.
// NewSlice panics if j < i.
NewSlice(i, j int64) Record
}
// simpleRecord is a basic, non-lazy in-memory record batch.
type simpleRecord struct {
refCount int64
schema *arrow.Schema
rows int64
arrs []Interface
}
// NewRecord returns a basic, non-lazy in-memory record batch.
//
// NewRecord panics if the columns and schema are inconsistent.
// NewRecord panics if rows is larger than the height of the columns.
func NewRecord(schema *arrow.Schema, cols []Interface, nrows int64) *simpleRecord {
rec := &simpleRecord{
refCount: 1,
schema: schema,
rows: nrows,
arrs: make([]Interface, len(cols)),
}
copy(rec.arrs, cols)
for _, arr := range rec.arrs {
arr.Retain()
}
if rec.rows < 0 {
switch len(rec.arrs) {
case 0:
rec.rows = 0
default:
rec.rows = int64(rec.arrs[0].Len())
}
}
err := rec.validate()
if err != nil {
rec.Release()
panic(err)
}
return rec
}
func (rec *simpleRecord) validate() error {
if len(rec.arrs) != len(rec.schema.Fields()) {
return fmt.Errorf("arrow/array: number of columns/fields mismatch")
}
for i, arr := range rec.arrs {
f := rec.schema.Field(i)
if int64(arr.Len()) < rec.rows {
return fmt.Errorf("arrow/array: mismatch number of rows in column %q: got=%d, want=%d",
f.Name,
arr.Len(), rec.rows,
)
}
if !arrow.TypeEqual(f.Type, arr.DataType()) {
return fmt.Errorf("arrow/array: column %q type mismatch: got=%v, want=%v",
f.Name,
arr.DataType(), f.Type,
)
}
}
return nil
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (rec *simpleRecord) Retain() {
atomic.AddInt64(&rec.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (rec *simpleRecord) Release() {
debug.Assert(atomic.LoadInt64(&rec.refCount) > 0, "too many releases")
if atomic.AddInt64(&rec.refCount, -1) == 0 {
for _, arr := range rec.arrs {
arr.Release()
}
rec.arrs = nil
}
}
func (rec *simpleRecord) Schema() *arrow.Schema { return rec.schema }
func (rec *simpleRecord) NumRows() int64 { return rec.rows }
func (rec *simpleRecord) NumCols() int64 { return int64(len(rec.arrs)) }
func (rec *simpleRecord) Columns() []Interface { return rec.arrs }
func (rec *simpleRecord) Column(i int) Interface { return rec.arrs[i] }
func (rec *simpleRecord) ColumnName(i int) string { return rec.schema.Field(i).Name }
// NewSlice constructs a zero-copy slice of the record with the indicated
// indices i and j, corresponding to array[i:j].
// The returned record must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the record array.
// NewSlice panics if j < i.
func (rec *simpleRecord) NewSlice(i, j int64) Record {
arrs := make([]Interface, len(rec.arrs))
for ii, arr := range rec.arrs {
arrs[ii] = NewSlice(arr, i, j)
}
defer func() {
for _, arr := range arrs {
arr.Release()
}
}()
return NewRecord(rec.schema, arrs, j-i)
}
func (rec *simpleRecord) String() string {
o := new(strings.Builder)
fmt.Fprintf(o, "record:\n %v\n", rec.schema)
fmt.Fprintf(o, " rows: %d\n", rec.rows)
for i, col := range rec.arrs {
fmt.Fprintf(o, " col[%d][%s]: %v\n", i, rec.schema.Field(i).Name, col)
}
return o.String()
}
// RecordBuilder eases the process of building a Record, iteratively, from
// a known Schema.
type RecordBuilder struct {
refCount int64
mem memory.Allocator
schema *arrow.Schema
fields []Builder
}
// NewRecordBuilder returns a builder, using the provided memory allocator and a schema.
func NewRecordBuilder(mem memory.Allocator, schema *arrow.Schema) *RecordBuilder {
b := &RecordBuilder{
refCount: 1,
mem: mem,
schema: schema,
fields: make([]Builder, len(schema.Fields())),
}
for i, f := range schema.Fields() {
b.fields[i] = NewBuilder(b.mem, f.Type)
}
return b
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *RecordBuilder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Release decreases the reference count by 1.
func (b *RecordBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
for _, f := range b.fields {
f.Release()
}
if atomic.AddInt64(&b.refCount, -1) == 0 {
b.fields = nil
}
}
func (b *RecordBuilder) Schema() *arrow.Schema { return b.schema }
func (b *RecordBuilder) Fields() []Builder { return b.fields }
func (b *RecordBuilder) Field(i int) Builder { return b.fields[i] }
func (b *RecordBuilder) Reserve(size int) {
for _, f := range b.fields {
f.Reserve(size)
}
}
// NewRecord creates a new record from the memory buffers and resets the
// RecordBuilder so it can be used to build a new record.
//
// The returned Record must be Release()'d after use.
//
// NewRecord panics if the fields' builder do not have the same length.
func (b *RecordBuilder) NewRecord() Record {
cols := make([]Interface, len(b.fields))
rows := int64(0)
defer func(cols []Interface) {
for _, col := range cols {
if col == nil {
continue
}
col.Release()
}
}(cols)
for i, f := range b.fields {
cols[i] = f.NewArray()
irow := int64(cols[i].Len())
if i > 0 && irow != rows {
panic(fmt.Errorf("arrow/array: field %d has %d rows. want=%d", i, irow, rows))
}
rows = irow
}
return NewRecord(b.schema, cols, rows)
}
var (
_ Record = (*simpleRecord)(nil)
_ RecordReader = (*simpleRecords)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/string.go
Generated Vendored
-205
View File
@@ -1,205 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"math"
"strings"
"unsafe"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/memory"
)
const (
stringArrayMaximumCapacity = math.MaxInt32
)
// String represents an immutable sequence of variable-length UTF-8 strings.
type String struct {
array
offsets []int32
values string
}
// NewStringData constructs a new String array from data.
func NewStringData(data *Data) *String {
a := &String{}
a.refCount = 1
a.setData(data)
return a
}
// Reset resets the String with a different set of Data.
func (a *String) Reset(data *Data) {
a.setData(data)
}
// Value returns the slice at index i. This value should not be mutated.
func (a *String) Value(i int) string {
i = i + a.array.data.offset
return a.values[a.offsets[i]:a.offsets[i+1]]
}
// ValueOffset returns the offset of the value at index i.
func (a *String) ValueOffset(i int) int { return int(a.offsets[i]) }
func (a *String) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *String) setData(data *Data) {
if len(data.buffers) != 3 {
panic("arrow/array: len(data.buffers) != 3")
}
a.array.setData(data)
if vdata := data.buffers[2]; vdata != nil {
b := vdata.Bytes()
a.values = *(*string)(unsafe.Pointer(&b))
}
if offsets := data.buffers[1]; offsets != nil {
a.offsets = arrow.Int32Traits.CastFromBytes(offsets.Bytes())
}
}
func arrayEqualString(left, right *String) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A StringBuilder is used to build a String array using the Append methods.
type StringBuilder struct {
builder *BinaryBuilder
}
// NewStringBuilder creates a new StringBuilder.
func NewStringBuilder(mem memory.Allocator) *StringBuilder {
b := &StringBuilder{
builder: NewBinaryBuilder(mem, arrow.BinaryTypes.String),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *StringBuilder) Release() {
b.builder.Release()
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *StringBuilder) Retain() {
b.builder.Retain()
}
//
// Len returns the number of elements in the array builder.
func (b *StringBuilder) Len() int { return b.builder.Len() }
// Cap returns the total number of elements that can be stored without allocating additional memory.
func (b *StringBuilder) Cap() int { return b.builder.Cap() }
// NullN returns the number of null values in the array builder.
func (b *StringBuilder) NullN() int { return b.builder.NullN() }
// Append appends a string to the builder.
func (b *StringBuilder) Append(v string) {
b.builder.Append([]byte(v))
}
// AppendNull appends a null to the builder.
func (b *StringBuilder) AppendNull() {
b.builder.AppendNull()
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *StringBuilder) AppendValues(v []string, valid []bool) {
b.builder.AppendStringValues(v, valid)
}
// Value returns the string at index i.
func (b *StringBuilder) Value(i int) string {
return string(b.builder.Value(i))
}
func (b *StringBuilder) init(capacity int) {
b.builder.init(capacity)
}
func (b *StringBuilder) resize(newBits int, init func(int)) {
b.builder.resize(newBits, init)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *StringBuilder) Reserve(n int) {
b.builder.Reserve(n)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *StringBuilder) Resize(n int) {
b.builder.Resize(n)
}
// NewArray creates a String array from the memory buffers used by the builder and resets the StringBuilder
// so it can be used to build a new array.
func (b *StringBuilder) NewArray() Interface {
return b.NewStringArray()
}
// NewStringArray creates a String array from the memory buffers used by the builder and resets the StringBuilder
// so it can be used to build a new array.
func (b *StringBuilder) NewStringArray() (a *String) {
data := b.builder.newData()
a = NewStringData(data)
data.Release()
return
}
var (
_ Interface = (*String)(nil)
_ Builder = (*StringBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/struct.go
Generated Vendored
-278
View File
@@ -1,278 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// Struct represents an ordered sequence of relative types.
type Struct struct {
array
fields []Interface
}
// NewStructData returns a new Struct array value from data.
func NewStructData(data *Data) *Struct {
a := &Struct{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Struct) NumField() int { return len(a.fields) }
func (a *Struct) Field(i int) Interface { return a.fields[i] }
func (a *Struct) String() string {
o := new(strings.Builder)
o.WriteString("{")
structBitmap := a.NullBitmapBytes()
for i, v := range a.fields {
if i > 0 {
o.WriteString(" ")
}
if !bytes.Equal(structBitmap, v.NullBitmapBytes()) {
masked := a.newStructFieldWithParentValidityMask(i)
fmt.Fprintf(o, "%v", masked)
masked.Release()
continue
}
fmt.Fprintf(o, "%v", v)
}
o.WriteString("}")
return o.String()
}
// newStructFieldWithParentValidityMask returns the Interface at fieldIndex
// with a nullBitmapBytes adjusted according on the parent struct nullBitmapBytes.
// From the docs:
// "When reading the struct array the parent validity bitmap takes priority."
func (a *Struct) newStructFieldWithParentValidityMask(fieldIndex int) Interface {
field := a.Field(fieldIndex)
nullBitmapBytes := field.NullBitmapBytes()
maskedNullBitmapBytes := make([]byte, len(nullBitmapBytes))
copy(maskedNullBitmapBytes, nullBitmapBytes)
for i := 0; i < field.Len(); i++ {
if !a.IsValid(i) {
bitutil.ClearBit(maskedNullBitmapBytes, i)
}
}
data := NewSliceData(field.Data(), 0, int64(field.Len()))
defer data.Release()
bufs := make([]*memory.Buffer, len(data.buffers))
copy(bufs, data.buffers)
bufs[0].Release()
bufs[0] = memory.NewBufferBytes(maskedNullBitmapBytes)
data.buffers = bufs
maskedField := MakeFromData(data)
return maskedField
}
func (a *Struct) setData(data *Data) {
a.array.setData(data)
a.fields = make([]Interface, len(data.childData))
for i, child := range data.childData {
if data.offset != 0 || child.length != data.length {
sub := NewSliceData(child, int64(data.offset), int64(data.offset+data.length))
a.fields[i] = MakeFromData(sub)
sub.Release()
} else {
a.fields[i] = MakeFromData(child)
}
}
}
func arrayEqualStruct(left, right *Struct) bool {
for i, lf := range left.fields {
rf := right.fields[i]
if !ArrayEqual(lf, rf) {
return false
}
}
return true
}
func (a *Struct) Retain() {
a.array.Retain()
for _, f := range a.fields {
f.Retain()
}
}
func (a *Struct) Release() {
a.array.Release()
for _, f := range a.fields {
f.Release()
}
}
type StructBuilder struct {
builder
dtype arrow.DataType
fields []Builder
}
// NewStructBuilder returns a builder, using the provided memory allocator.
func NewStructBuilder(mem memory.Allocator, dtype *arrow.StructType) *StructBuilder {
b := &StructBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
fields: make([]Builder, len(dtype.Fields())),
}
for i, f := range dtype.Fields() {
b.fields[i] = NewBuilder(b.mem, f.Type)
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *StructBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
for _, f := range b.fields {
f.Release()
}
}
func (b *StructBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
if !v {
for _, f := range b.fields {
f.AppendNull()
}
}
}
func (b *StructBuilder) AppendValues(valids []bool) {
b.Reserve(len(valids))
b.builder.unsafeAppendBoolsToBitmap(valids, len(valids))
}
func (b *StructBuilder) AppendNull() { b.Append(false) }
func (b *StructBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *StructBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *StructBuilder) init(capacity int) {
b.builder.init(capacity)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *StructBuilder) Reserve(n int) {
b.builder.reserve(n, b.resizeHelper)
for _, f := range b.fields {
f.Reserve(n)
}
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *StructBuilder) Resize(n int) {
b.resizeHelper(n)
for _, f := range b.fields {
f.Resize(n)
}
}
func (b *StructBuilder) resizeHelper(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
}
}
func (b *StructBuilder) NumField() int { return len(b.fields) }
func (b *StructBuilder) FieldBuilder(i int) Builder { return b.fields[i] }
// NewArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder
// so it can be used to build a new array.
func (b *StructBuilder) NewArray() Interface {
return b.NewStructArray()
}
// NewStructArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder
// so it can be used to build a new array.
func (b *StructBuilder) NewStructArray() (a *Struct) {
data := b.newData()
a = NewStructData(data)
data.Release()
return
}
func (b *StructBuilder) newData() (data *Data) {
fields := make([]*Data, len(b.fields))
for i, f := range b.fields {
arr := f.NewArray()
defer arr.Release()
fields[i] = arr.Data()
}
data = NewData(
b.dtype, b.length,
[]*memory.Buffer{
b.nullBitmap,
nil, // FIXME(sbinet)
},
fields,
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*Struct)(nil)
_ Builder = (*StructBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/table.go
Generated Vendored
-455
View File
@@ -1,455 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"errors"
"fmt"
"math"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
)
// Table represents a logical sequence of chunked arrays.
type Table interface {
Schema() *arrow.Schema
NumRows() int64
NumCols() int64
Column(i int) *Column
Retain()
Release()
}
// Column is an immutable column data structure consisting of
// a field (type metadata) and a chunked data array.
type Column struct {
field arrow.Field
data *Chunked
}
// NewColumn returns a column from a field and a chunked data array.
//
// NewColumn panics if the field's data type is inconsistent with the data type
// of the chunked data array.
func NewColumn(field arrow.Field, chunks *Chunked) *Column {
col := Column{
field: field,
data: chunks,
}
col.data.Retain()
if !arrow.TypeEqual(col.data.DataType(), col.field.Type) {
col.data.Release()
panic("arrow/array: inconsistent data type")
}
return &col
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (col *Column) Retain() {
col.data.Retain()
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (col *Column) Release() {
col.data.Release()
}
func (col *Column) Len() int { return col.data.Len() }
func (col *Column) NullN() int { return col.data.NullN() }
func (col *Column) Data() *Chunked { return col.data }
func (col *Column) Field() arrow.Field { return col.field }
func (col *Column) Name() string { return col.field.Name }
func (col *Column) DataType() arrow.DataType { return col.field.Type }
// NewSlice returns a new zero-copy slice of the column with the indicated
// indices i and j, corresponding to the column's array[i:j].
// The returned column must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the column's array.
// NewSlice panics if j < i.
func (col *Column) NewSlice(i, j int64) *Column {
return &Column{
field: col.field,
data: col.data.NewSlice(i, j),
}
}
// Chunked manages a collection of primitives arrays as one logical large array.
type Chunked struct {
refCount int64 // refCount must be first in the struct for 64 bit alignment and sync/atomic (https://github.com/golang/go/issues/37262)
chunks []Interface
length int
nulls int
dtype arrow.DataType
}
// NewChunked returns a new chunked array from the slice of arrays.
//
// NewChunked panics if the chunks do not have the same data type.
func NewChunked(dtype arrow.DataType, chunks []Interface) *Chunked {
arr := &Chunked{
chunks: make([]Interface, len(chunks)),
refCount: 1,
dtype: dtype,
}
for i, chunk := range chunks {
if !arrow.TypeEqual(chunk.DataType(), dtype) {
panic("arrow/array: mismatch data type")
}
chunk.Retain()
arr.chunks[i] = chunk
arr.length += chunk.Len()
arr.nulls += chunk.NullN()
}
return arr
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (a *Chunked) Retain() {
atomic.AddInt64(&a.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (a *Chunked) Release() {
debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
if atomic.AddInt64(&a.refCount, -1) == 0 {
for _, arr := range a.chunks {
arr.Release()
}
a.chunks = nil
a.length = 0
a.nulls = 0
}
}
func (a *Chunked) Len() int { return a.length }
func (a *Chunked) NullN() int { return a.nulls }
func (a *Chunked) DataType() arrow.DataType { return a.dtype }
func (a *Chunked) Chunks() []Interface { return a.chunks }
func (a *Chunked) Chunk(i int) Interface { return a.chunks[i] }
// NewSlice constructs a zero-copy slice of the chunked array with the indicated
// indices i and j, corresponding to array[i:j].
// The returned chunked array must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the input array.
// NewSlice panics if j < i.
func (a *Chunked) NewSlice(i, j int64) *Chunked {
if j > int64(a.length) || i > j || i > int64(a.length) {
panic("arrow/array: index out of range")
}
var (
cur = 0
beg = i
sz = j - i
chunks = make([]Interface, 0, len(a.chunks))
)
for cur < len(a.chunks) && beg >= int64(a.chunks[cur].Len()) {
beg -= int64(a.chunks[cur].Len())
cur++
}
for cur < len(a.chunks) && sz > 0 {
arr := a.chunks[cur]
end := beg + sz
if end > int64(arr.Len()) {
end = int64(arr.Len())
}
chunks = append(chunks, NewSlice(arr, beg, end))
sz -= int64(arr.Len()) - beg
beg = 0
cur++
}
chunks = chunks[:len(chunks):len(chunks)]
defer func() {
for _, chunk := range chunks {
chunk.Release()
}
}()
return NewChunked(a.dtype, chunks)
}
// simpleTable is a basic, non-lazy in-memory table.
type simpleTable struct {
refCount int64
rows int64
cols []Column
schema *arrow.Schema
}
// NewTable returns a new basic, non-lazy in-memory table.
// If rows is negative, the number of rows will be inferred from the height
// of the columns.
//
// NewTable panics if the columns and schema are inconsistent.
// NewTable panics if rows is larger than the height of the columns.
func NewTable(schema *arrow.Schema, cols []Column, rows int64) *simpleTable {
tbl := simpleTable{
refCount: 1,
rows: rows,
cols: cols,
schema: schema,
}
if tbl.rows < 0 {
switch len(tbl.cols) {
case 0:
tbl.rows = 0
default:
tbl.rows = int64(tbl.cols[0].Len())
}
}
// validate the table and its constituents.
// note we retain the columns after having validated the table
// in case the validation fails and panics (and would otherwise leak
// a ref-count on the columns.)
tbl.validate()
for i := range tbl.cols {
tbl.cols[i].Retain()
}
return &tbl
}
// NewTableFromRecords returns a new basic, non-lazy in-memory table.
//
// NewTableFromRecords panics if the records and schema are inconsistent.
func NewTableFromRecords(schema *arrow.Schema, recs []Record) *simpleTable {
arrs := make([]Interface, len(recs))
cols := make([]Column, len(schema.Fields()))
defer func(cols []Column) {
for i := range cols {
cols[i].Release()
}
}(cols)
for i := range cols {
field := schema.Field(i)
for j, rec := range recs {
arrs[j] = rec.Column(i)
}
chunk := NewChunked(field.Type, arrs)
cols[i] = *NewColumn(field, chunk)
chunk.Release()
}
return NewTable(schema, cols, -1)
}
func (tbl *simpleTable) Schema() *arrow.Schema { return tbl.schema }
func (tbl *simpleTable) NumRows() int64 { return tbl.rows }
func (tbl *simpleTable) NumCols() int64 { return int64(len(tbl.cols)) }
func (tbl *simpleTable) Column(i int) *Column { return &tbl.cols[i] }
func (tbl *simpleTable) validate() {
if len(tbl.cols) != len(tbl.schema.Fields()) {
panic(errors.New("arrow/array: table schema mismatch"))
}
for i, col := range tbl.cols {
if !col.field.Equal(tbl.schema.Field(i)) {
panic(fmt.Errorf("arrow/array: column field %q is inconsistent with schema", col.Name()))
}
if int64(col.Len()) < tbl.rows {
panic(fmt.Errorf("arrow/array: column %q expected length >= %d but got length %d", col.Name(), tbl.rows, col.Len()))
}
}
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (tbl *simpleTable) Retain() {
atomic.AddInt64(&tbl.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (tbl *simpleTable) Release() {
debug.Assert(atomic.LoadInt64(&tbl.refCount) > 0, "too many releases")
if atomic.AddInt64(&tbl.refCount, -1) == 0 {
for i := range tbl.cols {
tbl.cols[i].Release()
}
tbl.cols = nil
}
}
// TableReader is a Record iterator over a (possibly chunked) Table
type TableReader struct {
refCount int64
tbl Table
cur int64 // current row
max int64 // total number of rows
rec Record // current Record
chksz int64 // chunk size
chunks []*Chunked
slots []int // chunk indices
offsets []int64 // chunk offsets
}
// NewTableReader returns a new TableReader to iterate over the (possibly chunked) Table.
// if chunkSize is <= 0, the biggest possible chunk will be selected.
func NewTableReader(tbl Table, chunkSize int64) *TableReader {
ncols := tbl.NumCols()
tr := &TableReader{
refCount: 1,
tbl: tbl,
cur: 0,
max: int64(tbl.NumRows()),
chksz: chunkSize,
chunks: make([]*Chunked, ncols),
slots: make([]int, ncols),
offsets: make([]int64, ncols),
}
tr.tbl.Retain()
if tr.chksz <= 0 {
tr.chksz = math.MaxInt64
}
for i := range tr.chunks {
col := tr.tbl.Column(i)
tr.chunks[i] = col.Data()
tr.chunks[i].Retain()
}
return tr
}
func (tr *TableReader) Schema() *arrow.Schema { return tr.tbl.Schema() }
func (tr *TableReader) Record() Record { return tr.rec }
func (tr *TableReader) Next() bool {
if tr.cur >= tr.max {
return false
}
if tr.rec != nil {
tr.rec.Release()
}
// determine the minimum contiguous slice across all columns
chunksz := imin64(tr.max, tr.chksz)
chunks := make([]Interface, len(tr.chunks))
for i := range chunks {
j := tr.slots[i]
chunk := tr.chunks[i].Chunk(j)
remain := int64(chunk.Len()) - tr.offsets[i]
if remain < chunksz {
chunksz = remain
}
chunks[i] = chunk
}
// slice the chunks, advance each chunk slot as appropriate.
batch := make([]Interface, len(tr.chunks))
for i, chunk := range chunks {
var slice Interface
offset := tr.offsets[i]
switch int64(chunk.Len()) - offset {
case chunksz:
tr.slots[i]++
tr.offsets[i] = 0
if offset > 0 {
// need to slice
slice = NewSlice(chunk, offset, offset+chunksz)
} else {
// no need to slice
slice = chunk
slice.Retain()
}
default:
tr.offsets[i] += chunksz
slice = NewSlice(chunk, offset, offset+chunksz)
}
batch[i] = slice
}
tr.cur += chunksz
tr.rec = NewRecord(tr.tbl.Schema(), batch, chunksz)
for _, arr := range batch {
arr.Release()
}
return true
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (tr *TableReader) Retain() {
atomic.AddInt64(&tr.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (tr *TableReader) Release() {
debug.Assert(atomic.LoadInt64(&tr.refCount) > 0, "too many releases")
if atomic.AddInt64(&tr.refCount, -1) == 0 {
tr.tbl.Release()
for _, chk := range tr.chunks {
chk.Release()
}
if tr.rec != nil {
tr.rec.Release()
}
tr.tbl = nil
tr.chunks = nil
tr.slots = nil
tr.offsets = nil
}
}
func imin64(a, b int64) int64 {
if a < b {
return a
}
return b
}
var (
_ Table = (*simpleTable)(nil)
_ RecordReader = (*TableReader)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/util.go
Generated Vendored
-24
View File
@@ -1,24 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
func min(a, b int) int {
if a < b {
return a
}
return b
}
vendor/github.com/apache/arrow/go/arrow/arrio/arrio.go
Generated Vendored
-91
View File
@@ -1,91 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package arrio exposes functions to manipulate records, exposing and using
// interfaces not unlike the ones defined in the stdlib io package.
package arrio // import "github.com/apache/arrow/go/arrow/arrio"
import (
"io"
"github.com/apache/arrow/go/arrow/array"
)
// Reader is the interface that wraps the Read method.
type Reader interface {
// Read reads the current record from the underlying stream and an error, if any.
// When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
Read() (array.Record, error)
}
// ReaderAt is the interface that wraps the ReadAt method.
type ReaderAt interface {
// ReadAt reads the i-th record from the underlying stream and an error, if any.
ReadAt(i int64) (array.Record, error)
}
// Writer is the interface that wraps the Write method.
type Writer interface {
Write(rec array.Record) error
}
// Copy copies all the records available from src to dst.
// Copy returns the number of records copied and the first error
// encountered while copying, if any.
//
// A successful Copy returns err == nil, not err == EOF. Because Copy is
// defined to read from src until EOF, it does not treat an EOF from Read as an
// error to be reported.
func Copy(dst Writer, src Reader) (n int64, err error) {
for {
rec, err := src.Read()
if err != nil {
if err == io.EOF {
return n, nil
}
return n, err
}
err = dst.Write(rec)
if err != nil {
return n, err
}
n++
}
}
// CopyN copies n records (or until an error) from src to dst. It returns the
// number of records copied and the earliest error encountered while copying. On
// return, written == n if and only if err == nil.
func CopyN(dst Writer, src Reader, n int64) (written int64, err error) {
for ; written < n; written++ {
rec, err := src.Read()
if err != nil {
if err == io.EOF && written == n {
return written, nil
}
return written, err
}
err = dst.Write(rec)
if err != nil {
return written, err
}
}
if written != n && err == nil {
err = io.EOF
}
return written, err
}
vendor/github.com/apache/arrow/go/arrow/bitutil/bitutil.go
Generated Vendored
-156
View File
@@ -1,156 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bitutil
import (
"math/bits"
"reflect"
"unsafe"
)
var (
BitMask = [8]byte{1, 2, 4, 8, 16, 32, 64, 128}
FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
)
// IsMultipleOf8 returns whether v is a multiple of 8.
func IsMultipleOf8(v int64) bool { return v&7 == 0 }
func BytesForBits(bits int64) int64 { return (bits + 7) >> 3 }
// NextPowerOf2 rounds x to the next power of two.
func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) }
// CeilByte rounds size to the next multiple of 8.
func CeilByte(size int) int { return (size + 7) &^ 7 }
// CeilByte64 rounds size to the next multiple of 8.
func CeilByte64(size int64) int64 { return (size + 7) &^ 7 }
// BitIsSet returns true if the bit at index i in buf is set (1).
func BitIsSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) != 0 }
// BitIsNotSet returns true if the bit at index i in buf is not set (0).
func BitIsNotSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) == 0 }
// SetBit sets the bit at index i in buf to 1.
func SetBit(buf []byte, i int) { buf[uint(i)/8] |= BitMask[byte(i)%8] }
// ClearBit sets the bit at index i in buf to 0.
func ClearBit(buf []byte, i int) { buf[uint(i)/8] &= FlippedBitMask[byte(i)%8] }
// SetBitTo sets the bit at index i in buf to val.
func SetBitTo(buf []byte, i int, val bool) {
if val {
SetBit(buf, i)
} else {
ClearBit(buf, i)
}
}
// CountSetBits counts the number of 1's in buf up to n bits.
func CountSetBits(buf []byte, offset, n int) int {
if offset > 0 {
return countSetBitsWithOffset(buf, offset, n)
}
count := 0
uint64Bytes := n / uint64SizeBits * 8
for _, v := range bytesToUint64(buf[:uint64Bytes]) {
count += bits.OnesCount64(v)
}
for _, v := range buf[uint64Bytes : n/8] {
count += bits.OnesCount8(v)
}
// tail bits
for i := n &^ 0x7; i < n; i++ {
if BitIsSet(buf, i) {
count++
}
}
return count
}
func countSetBitsWithOffset(buf []byte, offset, n int) int {
count := 0
beg := offset
end := offset + n
begU8 := roundUp(beg, uint64SizeBits)
init := min(n, begU8-beg)
for i := offset; i < beg+init; i++ {
if BitIsSet(buf, i) {
count++
}
}
nU64 := (n - init) / uint64SizeBits
begU64 := begU8 / uint64SizeBits
endU64 := begU64 + nU64
bufU64 := bytesToUint64(buf)
if begU64 < len(bufU64) {
for _, v := range bufU64[begU64:endU64] {
count += bits.OnesCount64(v)
}
}
// FIXME: use a fallback to bits.OnesCount8
// before counting the tail bits.
tail := beg + init + nU64*uint64SizeBits
for i := tail; i < end; i++ {
if BitIsSet(buf, i) {
count++
}
}
return count
}
func roundUp(v, f int) int {
return (v + (f - 1)) / f * f
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
const (
uint64SizeBytes = int(unsafe.Sizeof(uint64(0)))
uint64SizeBits = uint64SizeBytes * 8
)
func bytesToUint64(b []byte) []uint64 {
h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
var res []uint64
s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
s.Data = h.Data
s.Len = h.Len / uint64SizeBytes
s.Cap = h.Cap / uint64SizeBytes
return res
}
vendor/github.com/apache/arrow/go/arrow/compare.go
Generated Vendored
-79
View File
@@ -1,79 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"reflect"
)
type typeEqualsConfig struct {
metadata bool
}
// TypeEqualOption is a functional option type used for configuring type
// equality checks.
type TypeEqualOption func(*typeEqualsConfig)
// CheckMetadata is an option for TypeEqual that allows checking for metadata
// equality besides type equality. It only makes sense for STRUCT type.
func CheckMetadata() TypeEqualOption {
return func(cfg *typeEqualsConfig) {
cfg.metadata = true
}
}
// TypeEqual checks if two DataType are the same, optionally checking metadata
// equality for STRUCT types.
func TypeEqual(left, right DataType, opts ...TypeEqualOption) bool {
var cfg typeEqualsConfig
for _, opt := range opts {
opt(&cfg)
}
switch {
case left == nil || right == nil:
return false
case left.ID() != right.ID():
return false
}
// StructType is the only type that has metadata.
l, ok := left.(*StructType)
if !ok || cfg.metadata {
return reflect.DeepEqual(left, right)
}
r := right.(*StructType)
switch {
case len(l.fields) != len(r.fields):
return false
case !reflect.DeepEqual(l.index, r.index):
return false
}
for i := range l.fields {
leftField, rightField := l.fields[i], r.fields[i]
switch {
case leftField.Name != rightField.Name:
return false
case leftField.Nullable != rightField.Nullable:
return false
case !TypeEqual(leftField.Type, rightField.Type, opts...):
return false
}
}
return true
}
vendor/github.com/apache/arrow/go/arrow/datatype.go
Generated Vendored
-143
View File
@@ -1,143 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
// Type is a logical type. They can be expressed as
// either a primitive physical type (bytes or bits of some fixed size), a
// nested type consisting of other data types, or another data type (e.g. a
// timestamp encoded as an int64)
type Type int
const (
// NULL type having no physical storage
NULL Type = iota
// BOOL is a 1 bit, LSB bit-packed ordering
BOOL
// UINT8 is an Unsigned 8-bit little-endian integer
UINT8
// INT8 is a Signed 8-bit little-endian integer
INT8
// UINT16 is an Unsigned 16-bit little-endian integer
UINT16
// INT16 is a Signed 16-bit little-endian integer
INT16
// UINT32 is an Unsigned 32-bit little-endian integer
UINT32
// INT32 is a Signed 32-bit little-endian integer
INT32
// UINT64 is an Unsigned 64-bit little-endian integer
UINT64
// INT64 is a Signed 64-bit little-endian integer
INT64
// FLOAT16 is a 2-byte floating point value
FLOAT16
// FLOAT32 is a 4-byte floating point value
FLOAT32
// FLOAT64 is an 8-byte floating point value
FLOAT64
// STRING is a UTF8 variable-length string
STRING
// BINARY is a Variable-length byte type (no guarantee of UTF8-ness)
BINARY
// FIXED_SIZE_BINARY is a binary where each value occupies the same number of bytes
FIXED_SIZE_BINARY
// DATE32 is int32 days since the UNIX epoch
DATE32
// DATE64 is int64 milliseconds since the UNIX epoch
DATE64
// TIMESTAMP is an exact timestamp encoded with int64 since UNIX epoch
// Default unit millisecond
TIMESTAMP
// TIME32 is a signed 32-bit integer, representing either seconds or
// milliseconds since midnight
TIME32
// TIME64 is a signed 64-bit integer, representing either microseconds or
// nanoseconds since midnight
TIME64
// INTERVAL is YEAR_MONTH or DAY_TIME interval in SQL style
INTERVAL
// DECIMAL is a precision- and scale-based decimal type. Storage type depends on the
// parameters.
DECIMAL
// LIST is a list of some logical data type
LIST
// STRUCT of logical types
STRUCT
// UNION of logical types
UNION
// DICTIONARY aka Category type
DICTIONARY
// MAP is a repeated struct logical type
MAP
// Custom data type, implemented by user
EXTENSION
// Fixed size list of some logical type
FIXED_SIZE_LIST
// Measure of elapsed time in either seconds, milliseconds, microseconds
// or nanoseconds.
DURATION
)
// DataType is the representation of an Arrow type.
type DataType interface {
ID() Type
// Name is name of the data type.
Name() string
}
// FixedWidthDataType is the representation of an Arrow type that
// requires a fixed number of bits in memory for each element.
type FixedWidthDataType interface {
DataType
// BitWidth returns the number of bits required to store a single element of this data type in memory.
BitWidth() int
}
type BinaryDataType interface {
DataType
binary()
}
vendor/github.com/apache/arrow/go/arrow/datatype_binary.go
Generated Vendored
-41
View File
@@ -1,41 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
type BinaryType struct{}
func (t *BinaryType) ID() Type { return BINARY }
func (t *BinaryType) Name() string { return "binary" }
func (t *BinaryType) String() string { return "binary" }
func (t *BinaryType) binary() {}
type StringType struct{}
func (t *StringType) ID() Type { return STRING }
func (t *StringType) Name() string { return "utf8" }
func (t *StringType) String() string { return "utf8" }
func (t *StringType) binary() {}
var (
BinaryTypes = struct {
Binary BinaryDataType
String BinaryDataType
}{
Binary: &BinaryType{},
String: &StringType{},
}
)
vendor/github.com/apache/arrow/go/arrow/datatype_fixedwidth.go
Generated Vendored
-213
View File
@@ -1,213 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"fmt"
"strconv"
)
type BooleanType struct{}
func (t *BooleanType) ID() Type { return BOOL }
func (t *BooleanType) Name() string { return "bool" }
func (t *BooleanType) String() string { return "bool" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *BooleanType) BitWidth() int { return 1 }
type FixedSizeBinaryType struct {
ByteWidth int
}
func (*FixedSizeBinaryType) ID() Type { return FIXED_SIZE_BINARY }
func (*FixedSizeBinaryType) Name() string { return "fixed_size_binary" }
func (t *FixedSizeBinaryType) BitWidth() int { return 8 * t.ByteWidth }
func (t *FixedSizeBinaryType) String() string {
return "fixed_size_binary[" + strconv.Itoa(t.ByteWidth) + "]"
}
type (
Timestamp int64
Time32 int32
Time64 int64
TimeUnit int
Date32 int32
Date64 int64
Duration int64
)
const (
Nanosecond TimeUnit = iota
Microsecond
Millisecond
Second
)
func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms", "s"}[uint(u)&3] }
// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z).
// The zero-value is a nanosecond and time zone neutral. Time zone neutral can be
// considered UTC without having "UTC" as a time zone.
type TimestampType struct {
Unit TimeUnit
TimeZone string
}
func (*TimestampType) ID() Type { return TIMESTAMP }
func (*TimestampType) Name() string { return "timestamp" }
func (t *TimestampType) String() string {
switch len(t.TimeZone) {
case 0:
return "timestamp[" + t.Unit.String() + "]"
default:
return "timestamp[" + t.Unit.String() + ", tz=" + t.TimeZone + "]"
}
}
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (*TimestampType) BitWidth() int { return 64 }
// Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight.
type Time32Type struct {
Unit TimeUnit
}
func (*Time32Type) ID() Type { return TIME32 }
func (*Time32Type) Name() string { return "time32" }
func (*Time32Type) BitWidth() int { return 32 }
func (t *Time32Type) String() string { return "time32[" + t.Unit.String() + "]" }
// Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight.
type Time64Type struct {
Unit TimeUnit
}
func (*Time64Type) ID() Type { return TIME64 }
func (*Time64Type) Name() string { return "time64" }
func (*Time64Type) BitWidth() int { return 64 }
func (t *Time64Type) String() string { return "time64[" + t.Unit.String() + "]" }
// DurationType is encoded as a 64-bit signed integer, representing an amount
// of elapsed time without any relation to a calendar artifact.
type DurationType struct {
Unit TimeUnit
}
func (*DurationType) ID() Type { return DURATION }
func (*DurationType) Name() string { return "duration" }
func (*DurationType) BitWidth() int { return 64 }
func (t *DurationType) String() string { return "duration[" + t.Unit.String() + "]" }
// Float16Type represents a floating point value encoded with a 16-bit precision.
type Float16Type struct{}
func (t *Float16Type) ID() Type { return FLOAT16 }
func (t *Float16Type) Name() string { return "float16" }
func (t *Float16Type) String() string { return "float16" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *Float16Type) BitWidth() int { return 16 }
// Decimal128Type represents a fixed-size 128-bit decimal type.
type Decimal128Type struct {
Precision int32
Scale int32
}
func (*Decimal128Type) ID() Type { return DECIMAL }
func (*Decimal128Type) Name() string { return "decimal" }
func (*Decimal128Type) BitWidth() int { return 16 }
func (t *Decimal128Type) String() string {
return fmt.Sprintf("%s(%d, %d)", t.Name(), t.Precision, t.Scale)
}
// MonthInterval represents a number of months.
type MonthInterval int32
// MonthIntervalType is encoded as a 32-bit signed integer,
// representing a number of months.
type MonthIntervalType struct{}
func (*MonthIntervalType) ID() Type { return INTERVAL }
func (*MonthIntervalType) Name() string { return "month_interval" }
func (*MonthIntervalType) String() string { return "month_interval" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *MonthIntervalType) BitWidth() int { return 32 }
// DayTimeInterval represents a number of days and milliseconds (fraction of day).
type DayTimeInterval struct {
Days int32 `json:"days"`
Milliseconds int32 `json:"milliseconds"`
}
// DayTimeIntervalType is encoded as a pair of 32-bit signed integer,
// representing a number of days and milliseconds (fraction of day).
type DayTimeIntervalType struct{}
func (*DayTimeIntervalType) ID() Type { return INTERVAL }
func (*DayTimeIntervalType) Name() string { return "day_time_interval" }
func (*DayTimeIntervalType) String() string { return "day_time_interval" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *DayTimeIntervalType) BitWidth() int { return 64 }
var (
FixedWidthTypes = struct {
Boolean FixedWidthDataType
Date32 FixedWidthDataType
Date64 FixedWidthDataType
DayTimeInterval FixedWidthDataType
Duration_s FixedWidthDataType
Duration_ms FixedWidthDataType
Duration_us FixedWidthDataType
Duration_ns FixedWidthDataType
Float16 FixedWidthDataType
MonthInterval FixedWidthDataType
Time32s FixedWidthDataType
Time32ms FixedWidthDataType
Time64us FixedWidthDataType
Time64ns FixedWidthDataType
Timestamp_s FixedWidthDataType
Timestamp_ms FixedWidthDataType
Timestamp_us FixedWidthDataType
Timestamp_ns FixedWidthDataType
}{
Boolean: &BooleanType{},
Date32: &Date32Type{},
Date64: &Date64Type{},
DayTimeInterval: &DayTimeIntervalType{},
Duration_s: &DurationType{Unit: Second},
Duration_ms: &DurationType{Unit: Millisecond},
Duration_us: &DurationType{Unit: Microsecond},
Duration_ns: &DurationType{Unit: Nanosecond},
Float16: &Float16Type{},
MonthInterval: &MonthIntervalType{},
Time32s: &Time32Type{Unit: Second},
Time32ms: &Time32Type{Unit: Millisecond},
Time64us: &Time64Type{Unit: Microsecond},
Time64ns: &Time64Type{Unit: Nanosecond},
Timestamp_s: &TimestampType{Unit: Second, TimeZone: "UTC"},
Timestamp_ms: &TimestampType{Unit: Millisecond, TimeZone: "UTC"},
Timestamp_us: &TimestampType{Unit: Microsecond, TimeZone: "UTC"},
Timestamp_ns: &TimestampType{Unit: Nanosecond, TimeZone: "UTC"},
}
_ FixedWidthDataType = (*FixedSizeBinaryType)(nil)
)
vendor/github.com/apache/arrow/go/arrow/datatype_nested.go
Generated Vendored
-180
View File
@@ -1,180 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"fmt"
"reflect"
"strings"
)
// ListType describes a nested type in which each array slot contains
// a variable-size sequence of values, all having the same relative type.
type ListType struct {
elem DataType // DataType of the list's elements
}
// ListOf returns the list type with element type t.
// For example, if t represents int32, ListOf(t) represents []int32.
//
// ListOf panics if t is nil or invalid.
func ListOf(t DataType) *ListType {
if t == nil {
panic("arrow: nil DataType")
}
return &ListType{elem: t}
}
func (*ListType) ID() Type { return LIST }
func (*ListType) Name() string { return "list" }
func (t *ListType) String() string { return fmt.Sprintf("list<item: %v>", t.elem) }
// Elem returns the ListType's element type.
func (t *ListType) Elem() DataType { return t.elem }
// FixedSizeListType describes a nested type in which each array slot contains
// a fixed-size sequence of values, all having the same relative type.
type FixedSizeListType struct {
n int32 // number of elements in the list
elem DataType // DataType of the list's elements
}
// FixedSizeListOf returns the list type with element type t.
// For example, if t represents int32, FixedSizeListOf(10, t) represents [10]int32.
//
// FixedSizeListOf panics if t is nil or invalid.
// FixedSizeListOf panics if n is <= 0.
func FixedSizeListOf(n int32, t DataType) *FixedSizeListType {
if t == nil {
panic("arrow: nil DataType")
}
if n <= 0 {
panic("arrow: invalid size")
}
return &FixedSizeListType{elem: t, n: n}
}
func (*FixedSizeListType) ID() Type { return FIXED_SIZE_LIST }
func (*FixedSizeListType) Name() string { return "fixed_size_list" }
func (t *FixedSizeListType) String() string {
return fmt.Sprintf("fixed_size_list<item: %v>[%d]", t.elem, t.n)
}
// Elem returns the FixedSizeListType's element type.
func (t *FixedSizeListType) Elem() DataType { return t.elem }
// Len returns the FixedSizeListType's size.
func (t *FixedSizeListType) Len() int32 { return t.n }
// StructType describes a nested type parameterized by an ordered sequence
// of relative types, called its fields.
type StructType struct {
fields []Field
index map[string]int
meta Metadata
}
// StructOf returns the struct type with fields fs.
//
// StructOf panics if there are duplicated fields.
// StructOf panics if there is a field with an invalid DataType.
func StructOf(fs ...Field) *StructType {
n := len(fs)
if n == 0 {
return &StructType{}
}
t := &StructType{
fields: make([]Field, n),
index: make(map[string]int, n),
}
for i, f := range fs {
if f.Type == nil {
panic("arrow: field with nil DataType")
}
t.fields[i] = Field{
Name: f.Name,
Type: f.Type,
Nullable: f.Nullable,
Metadata: f.Metadata.clone(),
}
if _, dup := t.index[f.Name]; dup {
panic(fmt.Errorf("arrow: duplicate field with name %q", f.Name))
}
t.index[f.Name] = i
}
return t
}
func (*StructType) ID() Type { return STRUCT }
func (*StructType) Name() string { return "struct" }
func (t *StructType) String() string {
o := new(strings.Builder)
o.WriteString("struct<")
for i, f := range t.fields {
if i > 0 {
o.WriteString(", ")
}
o.WriteString(fmt.Sprintf("%s: %v", f.Name, f.Type))
}
o.WriteString(">")
return o.String()
}
func (t *StructType) Fields() []Field { return t.fields }
func (t *StructType) Field(i int) Field { return t.fields[i] }
func (t *StructType) FieldByName(name string) (Field, bool) {
i, ok := t.index[name]
if !ok {
return Field{}, false
}
return t.fields[i], true
}
type Field struct {
Name string // Field name
Type DataType // The field's data type
Nullable bool // Fields can be nullable
Metadata Metadata // The field's metadata, if any
}
func (f Field) HasMetadata() bool { return f.Metadata.Len() != 0 }
func (f Field) Equal(o Field) bool {
return reflect.DeepEqual(f, o)
}
func (f Field) String() string {
o := new(strings.Builder)
nullable := ""
if f.Nullable {
nullable = ", nullable"
}
fmt.Fprintf(o, "%s: type=%v%v", f.Name, f.Type, nullable)
if f.HasMetadata() {
fmt.Fprintf(o, "\n%*.smetadata: %v", len(f.Name)+2, "", f.Metadata)
}
return o.String()
}
var (
_ DataType = (*ListType)(nil)
_ DataType = (*StructType)(nil)
)
vendor/github.com/apache/arrow/go/arrow/datatype_null.go
Generated Vendored
-29
View File
@@ -1,29 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
// NullType describes a degenerate array, with zero physical storage.
type NullType struct{}
func (*NullType) ID() Type { return NULL }
func (*NullType) Name() string { return "null" }
func (*NullType) String() string { return "null" }
var (
Null *NullType
_ DataType = Null
)
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go
Generated Vendored
-134
View File
@@ -1,134 +0,0 @@
// Code generated by datatype_numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
type Int8Type struct{}
func (t *Int8Type) ID() Type { return INT8 }
func (t *Int8Type) Name() string { return "int8" }
func (t *Int8Type) String() string { return "int8" }
func (t *Int8Type) BitWidth() int { return 8 }
type Int16Type struct{}
func (t *Int16Type) ID() Type { return INT16 }
func (t *Int16Type) Name() string { return "int16" }
func (t *Int16Type) String() string { return "int16" }
func (t *Int16Type) BitWidth() int { return 16 }
type Int32Type struct{}
func (t *Int32Type) ID() Type { return INT32 }
func (t *Int32Type) Name() string { return "int32" }
func (t *Int32Type) String() string { return "int32" }
func (t *Int32Type) BitWidth() int { return 32 }
type Int64Type struct{}
func (t *Int64Type) ID() Type { return INT64 }
func (t *Int64Type) Name() string { return "int64" }
func (t *Int64Type) String() string { return "int64" }
func (t *Int64Type) BitWidth() int { return 64 }
type Uint8Type struct{}
func (t *Uint8Type) ID() Type { return UINT8 }
func (t *Uint8Type) Name() string { return "uint8" }
func (t *Uint8Type) String() string { return "uint8" }
func (t *Uint8Type) BitWidth() int { return 8 }
type Uint16Type struct{}
func (t *Uint16Type) ID() Type { return UINT16 }
func (t *Uint16Type) Name() string { return "uint16" }
func (t *Uint16Type) String() string { return "uint16" }
func (t *Uint16Type) BitWidth() int { return 16 }
type Uint32Type struct{}
func (t *Uint32Type) ID() Type { return UINT32 }
func (t *Uint32Type) Name() string { return "uint32" }
func (t *Uint32Type) String() string { return "uint32" }
func (t *Uint32Type) BitWidth() int { return 32 }
type Uint64Type struct{}
func (t *Uint64Type) ID() Type { return UINT64 }
func (t *Uint64Type) Name() string { return "uint64" }
func (t *Uint64Type) String() string { return "uint64" }
func (t *Uint64Type) BitWidth() int { return 64 }
type Float32Type struct{}
func (t *Float32Type) ID() Type { return FLOAT32 }
func (t *Float32Type) Name() string { return "float32" }
func (t *Float32Type) String() string { return "float32" }
func (t *Float32Type) BitWidth() int { return 32 }
type Float64Type struct{}
func (t *Float64Type) ID() Type { return FLOAT64 }
func (t *Float64Type) Name() string { return "float64" }
func (t *Float64Type) String() string { return "float64" }
func (t *Float64Type) BitWidth() int { return 64 }
type Date32Type struct{}
func (t *Date32Type) ID() Type { return DATE32 }
func (t *Date32Type) Name() string { return "date32" }
func (t *Date32Type) String() string { return "date32" }
func (t *Date32Type) BitWidth() int { return 32 }
type Date64Type struct{}
func (t *Date64Type) ID() Type { return DATE64 }
func (t *Date64Type) Name() string { return "date64" }
func (t *Date64Type) String() string { return "date64" }
func (t *Date64Type) BitWidth() int { return 64 }
var (
PrimitiveTypes = struct {
Int8 DataType
Int16 DataType
Int32 DataType
Int64 DataType
Uint8 DataType
Uint16 DataType
Uint32 DataType
Uint64 DataType
Float32 DataType
Float64 DataType
Date32 DataType
Date64 DataType
}{
Int8: &Int8Type{},
Int16: &Int16Type{},
Int32: &Int32Type{},
Int64: &Int64Type{},
Uint8: &Uint8Type{},
Uint16: &Uint16Type{},
Uint32: &Uint32Type{},
Uint64: &Uint64Type{},
Float32: &Float32Type{},
Float64: &Float64Type{},
Date32: &Date32Type{},
Date64: &Date64Type{},
}
)
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpl
Generated Vendored
-40
View File
@@ -1,40 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
{{range .In}}
type {{.Name}}Type struct {}
func (t *{{.Name}}Type) ID() Type { return {{.Name|upper}} }
func (t *{{.Name}}Type) Name() string { return "{{.Name|lower}}" }
func (t *{{.Name}}Type) String() string { return "{{.Name|lower}}" }
func (t *{{.Name}}Type) BitWidth() int { return {{.Size}} }
{{end}}
var (
PrimitiveTypes = struct {
{{range .In}}
{{.Name}} DataType
{{- end}}
}{
{{range .In}}
{{.Name}}: &{{.Name}}Type{},
{{- end}}
}
)
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpldata
Generated Vendored
-66
View File
@@ -1,66 +0,0 @@
[
{
"Name": "Int8",
"Type": "int8",
"Size": 8
},
{
"Name": "Int16",
"Type": "int16",
"Size": 16
},
{
"Name": "Int32",
"Type": "int32",
"Size": 32
},
{
"Name": "Int64",
"Type": "int64",
"Size": 64
},
{
"Name": "Uint8",
"Type": "uint8",
"Size": 8
},
{
"Name": "Uint16",
"Type": "uint16",
"Size": 16
},
{
"Name": "Uint32",
"Type": "uint32",
"Size": 32
},
{
"Name": "Uint64",
"Type": "uint64",
"Size": 64
},
{
"Name": "Float32",
"Type": "float32",
"Size": 32
},
{
"Name": "Float64",
"Type": "float64",
"Size": 64
},
{
"Name": "Date32",
"Type": "date32",
"QualifiedType": "arrow.Date32",
"InternalType": "int32",
"Size": 32
},
{
"Name": "Date64",
"Type": "date64",
"QualifiedType": "arrow.Date64",
"InternalType": "int64",
"Size": 64
}
]
vendor/github.com/apache/arrow/go/arrow/decimal128/decimal128.go
Generated Vendored
-73
View File
@@ -1,73 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package decimal128 // import "github.com/apache/arrow/go/arrow/decimal128"
var (
MaxDecimal128 = New(542101086242752217, 687399551400673280-1)
)
// Num represents a signed 128-bit integer in two's complement.
// Calculations wrap around and overflow is ignored.
//
// For a discussion of the algorithms, look at Knuth's volume 2,
// Semi-numerical Algorithms section 4.3.1.
//
// Adapted from the Apache ORC C++ implementation
type Num struct {
lo uint64 // low bits
hi int64 // high bits
}
// New returns a new signed 128-bit integer value.
func New(hi int64, lo uint64) Num {
return Num{lo: lo, hi: hi}
}
// FromU64 returns a new signed 128-bit integer value from the provided uint64 one.
func FromU64(v uint64) Num {
return New(0, v)
}
// FromI64 returns a new signed 128-bit integer value from the provided int64 one.
func FromI64(v int64) Num {
switch {
case v > 0:
return New(0, uint64(v))
case v < 0:
return New(-1, uint64(v))
default:
return Num{}
}
}
// LowBits returns the low bits of the two's complement representation of the number.
func (n Num) LowBits() uint64 { return n.lo }
// HighBits returns the high bits of the two's complement representation of the number.
func (n Num) HighBits() int64 { return n.hi }
// Sign returns:
//
// -1 if x < 0
// 0 if x == 0
// +1 if x > 0
func (n Num) Sign() int {
if n == (Num{}) {
return 0
}
return int(1 | (n.hi >> 63))
}
vendor/github.com/apache/arrow/go/arrow/doc.go
Generated Vendored
-39
View File
@@ -1,39 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package arrow provides an implementation of Apache Arrow.
Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized
language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic
operations on modern hardware. It also provides computational libraries and zero-copy streaming
messaging and inter-process communication.
Basics
The fundamental data structure in Arrow is an Array, which holds a sequence of values of the same type. An array
consists of memory holding the data and an additional validity bitmap that indicates if the corresponding entry in the
array is valid (not null). If the array has no null entries, it is possible to omit this bitmap.
*/
package arrow
//go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
//go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl
//go:generate go run ./gen-flatbuffers.go
// stringer
//go:generate stringer -type=Type
vendor/github.com/apache/arrow/go/arrow/float16/float16.go
Generated Vendored
-70
View File
@@ -1,70 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package float16 // import "github.com/apache/arrow/go/arrow/float16"
import (
"math"
"strconv"
)
// Num represents a half-precision floating point value (float16)
// stored on 16 bits.
//
// See https://en.wikipedia.org/wiki/Half-precision_floating-point_format for more informations.
type Num struct {
bits uint16
}
// New creates a new half-precision floating point value from the provided
// float32 value.
func New(f float32) Num {
b := math.Float32bits(f)
sn := uint16((b >> 31) & 0x1)
exp := (b >> 23) & 0xff
res := int16(exp) - 127 + 15
fc := uint16(b>>13) & 0x3ff
switch {
case exp == 0:
res = 0
case exp == 0xff:
res = 0x1f
case res > 0x1e:
res = 0x1f
fc = 0
case res < 0x01:
res = 0
fc = 0
}
return Num{bits: (sn << 15) | uint16(res<<10) | fc}
}
func (f Num) Float32() float32 {
sn := uint32((f.bits >> 15) & 0x1)
exp := (f.bits >> 10) & 0x1f
res := uint32(exp) + 127 - 15
fc := uint32(f.bits & 0x3ff)
switch {
case exp == 0:
res = 0
case exp == 0x1f:
res = 0xff
}
return math.Float32frombits((sn << 31) | (res << 23) | (fc << 13))
}
func (f Num) Uint16() uint16 { return f.bits }
func (f Num) String() string { return strconv.FormatFloat(float64(f.Float32()), 'g', -1, 32) }
vendor/github.com/apache/arrow/go/arrow/go.mod
Generated Vendored
-27
View File
@@ -1,27 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
module github.com/apache/arrow/go/arrow
go 1.12
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/google/flatbuffers v1.11.0
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.2.0
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
)
vendor/github.com/apache/arrow/go/arrow/go.sum
Generated Vendored
-10
View File
@@ -1,10 +0,0 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg=
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
vendor/github.com/apache/arrow/go/arrow/internal/cpu/README.md
Generated Vendored
-42
View File
@@ -1,42 +0,0 @@
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Package cpu
Copied from Go src/internal/cpu
## Extras
### Intel
The `INTEL_DISABLE_EXT` environment variable can control which CPU extensions are available for
the running process. It should be a comma-separate list of upper-case strings as follows
| Flag | Description |
| -------- | ----------- |
| `ALL` | Disable all CPU extensions and fall back to Go implementation |
| `AVX2` | Disable AVX2 optimizations |
| `AVX` | Disable AVX optimizations |
| `SSE` | Disable all SSE optimizations |
| `SSE4` | Disable SSE42, SSE41 optimizations |
| `SSSE3` | Disable supplemental SSE3 optimizations |
| `SSE3` | Disable SSE3 optimizations |
| `SSE2` | Disable SSE2 optimizations |
Any unrecognized flags will be ignored and therefore it is possible to leave the environment variable with a bogus value such as `NONE` when experimenting.
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu.go
Generated Vendored
-77
View File
@@ -1,77 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cpu implements processor feature detection
// used by the Go standard library.
package cpu
var X86 x86
// The booleans in x86 contain the correspondingly named cpuid feature bit.
// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers
// in addition to the cpuid feature bit being set.
// The struct is padded to avoid false sharing.
type x86 struct {
_ [CacheLineSize]byte
HasAES bool
HasADX bool
HasAVX bool
HasAVX2 bool
HasBMI1 bool
HasBMI2 bool
HasERMS bool
HasFMA bool
HasOSXSAVE bool
HasPCLMULQDQ bool
HasPOPCNT bool
HasSSE2 bool
HasSSE3 bool
HasSSSE3 bool
HasSSE41 bool
HasSSE42 bool
_ [CacheLineSize]byte
}
var PPC64 ppc64
// For ppc64x, it is safe to check only for ISA level starting on ISA v3.00,
// since there are no optional categories. There are some exceptions that also
// require kernel support to work (darn, scv), so there are capability bits for
// those as well. The minimum processor requirement is POWER8 (ISA 2.07), so we
// maintain some of the old capability checks for optional categories for
// safety.
// The struct is padded to avoid false sharing.
type ppc64 struct {
_ [CacheLineSize]byte
HasVMX bool // Vector unit (Altivec)
HasDFP bool // Decimal Floating Point unit
HasVSX bool // Vector-scalar unit
HasHTM bool // Hardware Transactional Memory
HasISEL bool // Integer select
HasVCRYPTO bool // Vector cryptography
HasHTMNOSC bool // HTM: kernel-aborted transaction in syscalls
HasDARN bool // Hardware random number generator (requires kernel enablement)
HasSCV bool // Syscall vectored (requires kernel enablement)
IsPOWER8 bool // ISA v2.07 (POWER8)
IsPOWER9 bool // ISA v3.00 (POWER9)
_ [CacheLineSize]byte
}
var ARM64 arm64
// The booleans in arm64 contain the correspondingly named cpu feature bit.
// The struct is padded to avoid false sharing.
type arm64 struct {
_ [CacheLineSize]byte
HasFP bool
HasASIMD bool
HasEVTSTRM bool
HasAES bool
HasPMULL bool
HasSHA1 bool
HasSHA2 bool
HasCRC32 bool
HasATOMICS bool
_ [CacheLineSize]byte
}
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.go
Generated Vendored
-107
View File
@@ -1,107 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build 386 amd64 amd64p32
package cpu
import (
"os"
"strings"
)
const CacheLineSize = 64
// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)
func init() {
maxID, _, _, _ := cpuid(0, 0)
if maxID < 1 {
return
}
_, _, ecx1, edx1 := cpuid(1, 0)
X86.HasSSE2 = isSet(26, edx1)
X86.HasSSE3 = isSet(0, ecx1)
X86.HasPCLMULQDQ = isSet(1, ecx1)
X86.HasSSSE3 = isSet(9, ecx1)
X86.HasFMA = isSet(12, ecx1)
X86.HasSSE41 = isSet(19, ecx1)
X86.HasSSE42 = isSet(20, ecx1)
X86.HasPOPCNT = isSet(23, ecx1)
X86.HasAES = isSet(25, ecx1)
X86.HasOSXSAVE = isSet(27, ecx1)
osSupportsAVX := false
// For XGETBV, OSXSAVE bit is required and sufficient.
if X86.HasOSXSAVE {
eax, _ := xgetbv()
// Check if XMM and YMM registers have OS support.
osSupportsAVX = isSet(1, eax) && isSet(2, eax)
}
X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
if maxID < 7 {
return
}
_, ebx7, _, _ := cpuid(7, 0)
X86.HasBMI1 = isSet(3, ebx7)
X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
X86.HasBMI2 = isSet(8, ebx7)
X86.HasERMS = isSet(9, ebx7)
X86.HasADX = isSet(19, ebx7)
// NOTE(sgc): added ability to disable extension via environment
checkEnvironment()
}
func checkEnvironment() {
if ext, ok := os.LookupEnv("INTEL_DISABLE_EXT"); ok {
exts := strings.Split(ext, ",")
for _, x := range exts {
switch x {
case "ALL":
X86.HasAVX2 = false
X86.HasAVX = false
X86.HasSSE42 = false
X86.HasSSE41 = false
X86.HasSSSE3 = false
X86.HasSSE3 = false
X86.HasSSE2 = false
case "AVX2":
X86.HasAVX2 = false
case "AVX":
X86.HasAVX = false
case "SSE":
X86.HasSSE42 = false
X86.HasSSE41 = false
X86.HasSSSE3 = false
X86.HasSSE3 = false
X86.HasSSE2 = false
case "SSE4":
X86.HasSSE42 = false
X86.HasSSE41 = false
case "SSSE3":
X86.HasSSSE3 = false
case "SSE3":
X86.HasSSE3 = false
case "SSE2":
X86.HasSSE2 = false
}
}
}
}
func isSet(bitpos uint, value uint32) bool {
return value&(1<<bitpos) != 0
}
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.s
Generated Vendored
-32
View File
@@ -1,32 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build 386 amd64 amd64p32
#include "textflag.h"
// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
TEXT ·cpuid(SB), NOSPLIT, $0-24
MOVL eaxArg+0(FP), AX
MOVL ecxArg+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func xgetbv() (eax, edx uint32)
TEXT ·xgetbv(SB),NOSPLIT,$0-8
#ifdef GOOS_nacl
// nacl does not support XGETBV.
MOVL $0, eax+0(FP)
MOVL $0, edx+4(FP)
#else
MOVL $0, CX
XGETBV
MOVL AX, eax+0(FP)
MOVL DX, edx+4(FP)
#endif
RET
vendor/github.com/apache/arrow/go/arrow/internal/debug/assert_off.go
Generated Vendored
-24
View File
@@ -1,24 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !assert
package debug
// Assert will panic with msg if cond is false.
//
// msg must be a string, func() string or fmt.Stringer.
func Assert(cond bool, msg interface{}) {}
vendor/github.com/apache/arrow/go/arrow/internal/debug/assert_on.go
Generated Vendored
-28
View File
@@ -1,28 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build assert
package debug
// Assert will panic with msg if cond is false.
//
// msg must be a string, func() string or fmt.Stringer.
func Assert(cond bool, msg interface{}) {
if !cond {
panic(getStringValue(msg))
}
}
vendor/github.com/apache/arrow/go/arrow/internal/debug/doc.go
Generated Vendored
-32
View File
@@ -1,32 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package debug provides APIs for conditional runtime assertions and debug logging.
Using Assert
To enable runtime assertions, build with the assert tag. When the assert tag is omitted,
the code for the assertion will be omitted from the binary.
Using Log
To enable runtime debug logs, build with the debug tag. When the debug tag is omitted,
the code for logging will be omitted from the binary.
*/
package debug
vendor/github.com/apache/arrow/go/arrow/internal/debug/log_off.go
Generated Vendored
-21
View File
@@ -1,21 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !debug
package debug
func Log(interface{}) {}
vendor/github.com/apache/arrow/go/arrow/internal/debug/log_on.go
Generated Vendored
-32
View File
@@ -1,32 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build debug
package debug
import (
"log"
"os"
)
var (
debug = log.New(os.Stderr, "[D] ", log.LstdFlags)
)
func Log(msg interface{}) {
debug.Output(1, getStringValue(msg))
}
vendor/github.com/apache/arrow/go/arrow/internal/debug/util.go
Generated Vendored
-37
View File
@@ -1,37 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build debug assert
package debug
import "fmt"
func getStringValue(v interface{}) string {
switch a := v.(type) {
case func() string:
return a()
case string:
return a
case fmt.Stringer:
return a.String()
default:
panic(fmt.Sprintf("unexpected type, %t", v))
}
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Binary.go
Generated Vendored
-51
View File
@@ -1,51 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// Opaque binary data
type Binary struct {
_tab flatbuffers.Table
}
func GetRootAsBinary(buf []byte, offset flatbuffers.UOffsetT) *Binary {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Binary{}
x.Init(buf, n+offset)
return x
}
func (rcv *Binary) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Binary) Table() flatbuffers.Table {
return rcv._tab
}
func BinaryStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func BinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Block.go
Generated Vendored
-74
View File
@@ -1,74 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Block struct {
_tab flatbuffers.Struct
}
func (rcv *Block) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Block) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// Index to the start of the RecordBlock (note this is past the Message header)
func (rcv *Block) Offset() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// Index to the start of the RecordBlock (note this is past the Message header)
func (rcv *Block) MutateOffset(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// Length of the metadata
func (rcv *Block) MetaDataLength() int32 {
return rcv._tab.GetInt32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// Length of the metadata
func (rcv *Block) MutateMetaDataLength(n int32) bool {
return rcv._tab.MutateInt32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
/// Length of the data (this is aligned so there can be a gap between this and
/// the metatdata).
func (rcv *Block) BodyLength() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(16))
}
/// Length of the data (this is aligned so there can be a gap between this and
/// the metatdata).
func (rcv *Block) MutateBodyLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n)
}
func CreateBlock(builder *flatbuffers.Builder, offset int64, metaDataLength int32, bodyLength int64) flatbuffers.UOffsetT {
builder.Prep(8, 24)
builder.PrependInt64(bodyLength)
builder.Pad(4)
builder.PrependInt32(metaDataLength)
builder.PrependInt64(offset)
return builder.Offset()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Bool.go
Generated Vendored
-50
View File
@@ -1,50 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Bool struct {
_tab flatbuffers.Table
}
func GetRootAsBool(buf []byte, offset flatbuffers.UOffsetT) *Bool {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Bool{}
x.Init(buf, n+offset)
return x
}
func (rcv *Bool) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Bool) Table() flatbuffers.Table {
return rcv._tab
}
func BoolStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func BoolEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Buffer.go
Generated Vendored
-73
View File
@@ -1,73 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// A Buffer represents a single contiguous memory segment
type Buffer struct {
_tab flatbuffers.Struct
}
func (rcv *Buffer) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Buffer) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// The relative offset into the shared memory page where the bytes for this
/// buffer starts
func (rcv *Buffer) Offset() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// The relative offset into the shared memory page where the bytes for this
/// buffer starts
func (rcv *Buffer) MutateOffset(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// The absolute length (in bytes) of the memory buffer. The memory is found
/// from offset (inclusive) to offset + length (non-inclusive). When building
/// messages using the encapsulated IPC message, padding bytes may be written
/// after a buffer, but such padding bytes do not need to be accounted for in
/// the size here.
func (rcv *Buffer) Length() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// The absolute length (in bytes) of the memory buffer. The memory is found
/// from offset (inclusive) to offset + length (non-inclusive). When building
/// messages using the encapsulated IPC message, padding bytes may be written
/// after a buffer, but such padding bytes do not need to be accounted for in
/// the size here.
func (rcv *Buffer) MutateLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
func CreateBuffer(builder *flatbuffers.Builder, offset int64, length int64) flatbuffers.UOffsetT {
builder.Prep(8, 16)
builder.PrependInt64(length)
builder.PrependInt64(offset)
return builder.Offset()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Date.go
Generated Vendored
-71
View File
@@ -1,71 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
/// epoch (1970-01-01), stored in either of two units:
///
/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
/// leap seconds), where the values are evenly divisible by 86400000
/// * Days (32 bits) since the UNIX epoch
type Date struct {
_tab flatbuffers.Table
}
func GetRootAsDate(buf []byte, offset flatbuffers.UOffsetT) *Date {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Date{}
x.Init(buf, n+offset)
return x
}
func (rcv *Date) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Date) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Date) Unit() DateUnit {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 1
}
func (rcv *Date) MutateUnit(n DateUnit) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func DateStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func DateAddUnit(builder *flatbuffers.Builder, unit int16) {
builder.PrependInt16Slot(0, unit, 1)
}
func DateEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DateUnit.go
Generated Vendored
-31
View File
@@ -1,31 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
type DateUnit = int16
const (
DateUnitDAY DateUnit = 0
DateUnitMILLISECOND DateUnit = 1
)
var EnumNamesDateUnit = map[DateUnit]string{
DateUnitDAY:"DAY",
DateUnitMILLISECOND:"MILLISECOND",
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Decimal.go
Generated Vendored
-84
View File
@@ -1,84 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Decimal struct {
_tab flatbuffers.Table
}
func GetRootAsDecimal(buf []byte, offset flatbuffers.UOffsetT) *Decimal {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Decimal{}
x.Init(buf, n+offset)
return x
}
func (rcv *Decimal) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Decimal) Table() flatbuffers.Table {
return rcv._tab
}
/// Total number of decimal digits
func (rcv *Decimal) Precision() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Total number of decimal digits
func (rcv *Decimal) MutatePrecision(n int32) bool {
return rcv._tab.MutateInt32Slot(4, n)
}
/// Number of digits after the decimal point "."
func (rcv *Decimal) Scale() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Number of digits after the decimal point "."
func (rcv *Decimal) MutateScale(n int32) bool {
return rcv._tab.MutateInt32Slot(6, n)
}
func DecimalStart(builder *flatbuffers.Builder) {
builder.StartObject(2)
}
func DecimalAddPrecision(builder *flatbuffers.Builder, precision int32) {
builder.PrependInt32Slot(0, precision, 0)
}
func DecimalAddScale(builder *flatbuffers.Builder, scale int32) {
builder.PrependInt32Slot(1, scale, 0)
}
func DecimalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DictionaryBatch.go
Generated Vendored
-106
View File
@@ -1,106 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// For sending dictionary encoding information. Any Field can be
/// dictionary-encoded, but in this case none of its children may be
/// dictionary-encoded.
/// There is one vector / column per dictionary, but that vector / column
/// may be spread across multiple dictionary batches by using the isDelta
/// flag
type DictionaryBatch struct {
_tab flatbuffers.Table
}
func GetRootAsDictionaryBatch(buf []byte, offset flatbuffers.UOffsetT) *DictionaryBatch {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &DictionaryBatch{}
x.Init(buf, n+offset)
return x
}
func (rcv *DictionaryBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *DictionaryBatch) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *DictionaryBatch) Id() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
func (rcv *DictionaryBatch) MutateId(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
func (rcv *DictionaryBatch) Data(obj *RecordBatch) *RecordBatch {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(RecordBatch)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// If isDelta is true the values in the dictionary are to be appended to a
/// dictionary with the indicated id
func (rcv *DictionaryBatch) IsDelta() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// If isDelta is true the values in the dictionary are to be appended to a
/// dictionary with the indicated id
func (rcv *DictionaryBatch) MutateIsDelta(n bool) bool {
return rcv._tab.MutateBoolSlot(8, n)
}
func DictionaryBatchStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func DictionaryBatchAddId(builder *flatbuffers.Builder, id int64) {
builder.PrependInt64Slot(0, id, 0)
}
func DictionaryBatchAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(data), 0)
}
func DictionaryBatchAddIsDelta(builder *flatbuffers.Builder, isDelta bool) {
builder.PrependBoolSlot(2, isDelta, false)
}
func DictionaryBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DictionaryEncoding.go
Generated Vendored
-116
View File
@@ -1,116 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// Dictionary encoding metadata
type DictionaryEncoding struct {
_tab flatbuffers.Table
}
func GetRootAsDictionaryEncoding(buf []byte, offset flatbuffers.UOffsetT) *DictionaryEncoding {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &DictionaryEncoding{}
x.Init(buf, n+offset)
return x
}
func (rcv *DictionaryEncoding) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *DictionaryEncoding) Table() flatbuffers.Table {
return rcv._tab
}
/// The known dictionary id in the application where this data is used. In
/// the file or streaming formats, the dictionary ids are found in the
/// DictionaryBatch messages
func (rcv *DictionaryEncoding) Id() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
/// The known dictionary id in the application where this data is used. In
/// the file or streaming formats, the dictionary ids are found in the
/// DictionaryBatch messages
func (rcv *DictionaryEncoding) MutateId(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
/// The dictionary indices are constrained to be positive integers. If this
/// field is null, the indices must be signed int32
func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(Int)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// The dictionary indices are constrained to be positive integers. If this
/// field is null, the indices must be signed int32
/// By default, dictionaries are not ordered, or the order does not have
/// semantic meaning. In some statistical, applications, dictionary-encoding
/// is used to represent ordered categorical data, and we provide a way to
/// preserve that metadata here
func (rcv *DictionaryEncoding) IsOrdered() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// By default, dictionaries are not ordered, or the order does not have
/// semantic meaning. In some statistical, applications, dictionary-encoding
/// is used to represent ordered categorical data, and we provide a way to
/// preserve that metadata here
func (rcv *DictionaryEncoding) MutateIsOrdered(n bool) bool {
return rcv._tab.MutateBoolSlot(8, n)
}
func DictionaryEncodingStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func DictionaryEncodingAddId(builder *flatbuffers.Builder, id int64) {
builder.PrependInt64Slot(0, id, 0)
}
func DictionaryEncodingAddIndexType(builder *flatbuffers.Builder, indexType flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(indexType), 0)
}
func DictionaryEncodingAddIsOrdered(builder *flatbuffers.Builder, isOrdered bool) {
builder.PrependBoolSlot(2, isOrdered, false)
}
func DictionaryEncodingEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Duration.go
Generated Vendored
-65
View File
@@ -1,65 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Duration struct {
_tab flatbuffers.Table
}
func GetRootAsDuration(buf []byte, offset flatbuffers.UOffsetT) *Duration {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Duration{}
x.Init(buf, n+offset)
return x
}
func (rcv *Duration) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Duration) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Duration) Unit() TimeUnit {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 1
}
func (rcv *Duration) MutateUnit(n TimeUnit) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func DurationStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func DurationAddUnit(builder *flatbuffers.Builder, unit int16) {
builder.PrependInt16Slot(0, unit, 1)
}
func DurationEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Endianness.go
Generated Vendored
-33
View File
@@ -1,33 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
/// ----------------------------------------------------------------------
/// Endianness of the platform producing the data
type Endianness = int16
const (
EndiannessLittle Endianness = 0
EndiannessBig Endianness = 1
)
var EnumNamesEndianness = map[Endianness]string{
EndiannessLittle:"Little",
EndiannessBig:"Big",
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Field.go
Generated Vendored
-188
View File
@@ -1,188 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// A field represents a named column in a record / row batch or child of a
/// nested type.
type Field struct {
_tab flatbuffers.Table
}
func GetRootAsField(buf []byte, offset flatbuffers.UOffsetT) *Field {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Field{}
x.Init(buf, n+offset)
return x
}
func (rcv *Field) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Field) Table() flatbuffers.Table {
return rcv._tab
}
/// Name is not required, in i.e. a List
func (rcv *Field) Name() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
/// Name is not required, in i.e. a List
/// Whether or not this field can contain nulls. Should be true in general.
func (rcv *Field) Nullable() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// Whether or not this field can contain nulls. Should be true in general.
func (rcv *Field) MutateNullable(n bool) bool {
return rcv._tab.MutateBoolSlot(6, n)
}
func (rcv *Field) TypeType() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Field) MutateTypeType(n byte) bool {
return rcv._tab.MutateByteSlot(8, n)
}
/// This is the type of the decoded value if the field is dictionary encoded.
func (rcv *Field) Type(obj *flatbuffers.Table) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
rcv._tab.Union(obj, o)
return true
}
return false
}
/// This is the type of the decoded value if the field is dictionary encoded.
/// Present only if the field is dictionary encoded.
func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(DictionaryEncoding)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// Present only if the field is dictionary encoded.
/// children apply only to nested data types like Struct, List and Union. For
/// primitive types children will have length 0.
func (rcv *Field) Children(obj *Field, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Field) ChildrenLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// children apply only to nested data types like Struct, List and Union. For
/// primitive types children will have length 0.
/// User-defined metadata
func (rcv *Field) CustomMetadata(obj *KeyValue, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Field) CustomMetadataLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// User-defined metadata
func FieldStart(builder *flatbuffers.Builder) {
builder.StartObject(7)
}
func FieldAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
}
func FieldAddNullable(builder *flatbuffers.Builder, nullable bool) {
builder.PrependBoolSlot(1, nullable, false)
}
func FieldAddTypeType(builder *flatbuffers.Builder, typeType byte) {
builder.PrependByteSlot(2, typeType, 0)
}
func FieldAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(type_), 0)
}
func FieldAddDictionary(builder *flatbuffers.Builder, dictionary flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(dictionary), 0)
}
func FieldAddChildren(builder *flatbuffers.Builder, children flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(children), 0)
}
func FieldStartChildrenVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func FieldAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(customMetadata), 0)
}
func FieldStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func FieldEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FieldNode.go
Generated Vendored
-76
View File
@@ -1,76 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// Data structures for describing a table row batch (a collection of
/// equal-length Arrow arrays)
/// Metadata about a field at some level of a nested type tree (but not
/// its children).
///
/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
/// null_count: 0} for its Int16 node, as separate FieldNode structs
type FieldNode struct {
_tab flatbuffers.Struct
}
func (rcv *FieldNode) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *FieldNode) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// The number of value slots in the Arrow array at this level of a nested
/// tree
func (rcv *FieldNode) Length() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// The number of value slots in the Arrow array at this level of a nested
/// tree
func (rcv *FieldNode) MutateLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
func (rcv *FieldNode) NullCount() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
func (rcv *FieldNode) MutateNullCount(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
func CreateFieldNode(builder *flatbuffers.Builder, length int64, nullCount int64) flatbuffers.UOffsetT {
builder.Prep(8, 16)
builder.PrependInt64(nullCount)
builder.PrependInt64(length)
return builder.Offset()
}

Some files were not shown because too many files have changed in this diff Show More