diff --git a/go.mod b/go.mod index 28e966b25e3..d0ce7b4dc83 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,6 @@ require ( github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/buger/jsonparser v1.1.1 // indirect github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/centrifugal/protocol v0.7.6 // indirect @@ -153,7 +152,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/emicklei/proto v1.6.15 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect @@ -246,6 +244,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.4.0 github.com/Azure/go-autorest/autorest/adal v0.9.17 github.com/armon/go-radix v1.0.0 + github.com/blugelabs/bluge v0.1.9 github.com/golang-migrate/migrate/v4 v4.7.0 github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f github.com/grafana/thema v0.0.0-20220413232647-fc54c169b508 @@ -274,9 +273,21 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.2.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/RoaringBitmap/roaring v0.9.1 // indirect + github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/blevesearch/mmap-go v1.0.2 // indirect + github.com/blevesearch/segment v0.9.0 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/blevesearch/vellum v1.0.5 // indirect + github.com/blugelabs/bluge_segment_api v0.2.0 // indirect + github.com/blugelabs/ice v0.2.0 // indirect + github.com/caio/go-tdigest v3.1.0+incompatible // indirect github.com/chromedp/cdproto v0.0.0-20220208224320-6efb837e6bc2 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect + github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect github.com/elazarl/goproxy v0.0.0-20220115173737-adb46da277ac // indirect github.com/getkin/kin-openapi v0.94.0 // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect @@ -287,6 +298,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/klauspost/compress v1.15.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/segmentio/asm v1.1.1 // indirect diff --git a/go.sum b/go.sum index 073095a20d6..cf589f5fef9 100644 --- a/go.sum +++ b/go.sum @@ -293,6 +293,10 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RoaringBitmap/gocroaring v0.4.0/go.mod h1:NieMwz7ZqwU2DD73/vvYwv7r4eWBKuPVSXZIpsaMwCI= +github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76/go.mod h1:oM0MHmQ3nDsq609SS36p+oYbRi16+oVvU2Bw4Ipv0SE= +github.com/RoaringBitmap/roaring v0.9.1 h1:5PRizBmoN/PfV17nPNQou4dHQ7NcJi8FO/bihdYyCEM= +github.com/RoaringBitmap/roaring v0.9.1/go.mod h1:h1B7iIUOmnAeb5ytYMvnHJwxMc6LUrwBnzXWRuqTQUc= github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= @@ -442,6 +446,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4Nx github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f h1:y06x6vGnFYfXUoVMbrcP1Uzpj4JG01eB5vRps9G8agM= +github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f/go.mod h1:2stgcRjl6QmW+gU2h5E7BQXg4HU0gzxKWDuT5HviN9s= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -462,6 +468,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= @@ -469,6 +476,22 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= +github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blugelabs/bluge v0.1.9 h1:bPgXlcsWugrXNjzeoLdOnvfJpHsyODKpYaAndayl/SM= +github.com/blugelabs/bluge v0.1.9/go.mod h1:5d7LktUkQgvbh5Bmi6tPWtvo4+6uRTm6gAwP+5z6FqQ= +github.com/blugelabs/bluge_segment_api v0.2.0 h1:cCX1Y2y8v0LZ7+EEJ6gH7dW6TtVTW4RhG0vp3R+N2Lo= +github.com/blugelabs/bluge_segment_api v0.2.0/go.mod h1:95XA+ZXfRj/IXADm7gZ+iTcWOJPg5jQTY1EReIzl3LA= +github.com/blugelabs/ice v0.2.0 h1:9N/TRBqAr43emheD1ptk9mohuT6xAVq83gesgE60Qqk= +github.com/blugelabs/ice v0.2.0/go.mod h1:7foiDf4V83FIYYnGh2LOoRWsbNoCqAAMNgKn879Iyu0= github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= @@ -481,7 +504,6 @@ github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQ github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= @@ -491,6 +513,8 @@ github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9 github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/cactus/go-statsd-client/statsd v0.0.0-20191106001114-12b4e2b38748/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/caio/go-tdigest v2.3.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= +github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds= +github.com/caio/go-tdigest v3.1.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/casbin/casbin/v2 v2.31.6/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -762,6 +786,8 @@ github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -828,7 +854,6 @@ github.com/dop251/goja v0.0.0-20210804101310-32956a348b49/go.mod h1:R9ET47fwRVRP github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -1627,6 +1652,7 @@ github.com/influxdata/flux v0.120.1/go.mod h1:pGSAvyAA5d3et7SSzajaYShWYXmnRnJJq2 github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo= github.com/influxdata/go-syslog/v3 v3.0.1-0.20201128200927-a1889d947b48/go.mod h1:aXdIdfn2OcGnMhOTojXmwZqXKgC3MU5riiNvzwwG9OY= github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69/go.mod h1:pwymjR6SrP3gD3pRj9RJwdl1j5s3doEEV8gS4X9qSzA= +github.com/influxdata/influxdb v1.7.6/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.0/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ= github.com/influxdata/influxdb v1.8.1/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ= @@ -1843,6 +1869,7 @@ github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939/go.mod h1:dXGbAdH5GtB github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= @@ -2050,6 +2077,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9 github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/multiplay/go-ts3 v1.0.0/go.mod h1:14S6cS3fLNT3xOytrA/DkRyAFNuQLMLEqOYAsf87IbQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -3141,6 +3170,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -3470,6 +3500,7 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= diff --git a/pkg/services/searchV2/bluge.go b/pkg/services/searchV2/bluge.go new file mode 100644 index 00000000000..19543ac31f9 --- /dev/null +++ b/pkg/services/searchV2/bluge.go @@ -0,0 +1,541 @@ +package searchV2 + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/blugelabs/bluge" + "github.com/blugelabs/bluge/search" + "github.com/blugelabs/bluge/search/aggregations" + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/log" +) + +const ( + documentFieldUID = "_id" // actually UID!! but bluge likes "_id" + documentFieldKind = "kind" + documentFieldTag = "tag" + documentFieldURL = "url" + documentFieldName = "name" + documentFieldDescription = "description" + documentFieldLocation = "location" // parent path + documentFieldPanelType = "panel_type" + documentFieldDSUID = "ds_uid" + documentFieldDSType = "ds_type" + documentFieldInternalID = "__internal_id" // only for migrations! (indexed as a string) +) + +func initBlugeIndex(dashboards []dashboard, logger log.Logger) (*bluge.Reader, error) { + config := bluge.InMemoryOnlyConfig() + + // open an index writer using the configuration + writer, err := bluge.OpenWriter(config) + if err != nil { + return nil, fmt.Errorf("error opening writer: %v", err) + } + defer func() { + err = writer.Close() + if err != nil { + logger.Error("Error closing bluge writer", "error", err) + } + }() + + start := time.Now() + + logger.Info("Loading dashboards for bluge index", "elapsed", time.Since(start), "numDashboards", len(dashboards)) + label := time.Now() + + batch := bluge.NewBatch() + + // First index the folders + folderIdLookup := make(map[int64]string, 50) + for _, dashboard := range dashboards { + if !dashboard.isFolder { + continue + } + uid := dashboard.uid + url := fmt.Sprintf("/dashboards/f/%s/%s", dashboard.uid, dashboard.slug) + if uid == "" { + uid = "general" + url = "/dashboards" + dashboard.info.Title = "General" + dashboard.info.Description = "" + + // ARRRG, why is this not in the final index?!! + } + + doc := bluge.NewDocument(uid). + AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindFolder)).Aggregatable().StoreValue()). + AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()). + AddField(bluge.NewTextField(documentFieldName, dashboard.info.Title).StoreValue().SearchTermPositions()). + AddField(bluge.NewTextField(documentFieldDescription, dashboard.info.Description).SearchTermPositions()) + + batch.Insert(doc) + + folderIdLookup[dashboard.id] = uid + } + + // Then each dashboard + for _, dashboard := range dashboards { + if dashboard.isFolder { + continue + } + + url := fmt.Sprintf("/d/%s/%s", dashboard.uid, dashboard.slug) + folderUID := folderIdLookup[dashboard.folderID] + location := folderUID + + // Dashboard document + doc := bluge.NewDocument(dashboard.uid). + AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindDashboard)).Aggregatable().StoreValue()). + AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()). + AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()). + AddField(bluge.NewTextField(documentFieldName, dashboard.info.Title).StoreValue().SearchTermPositions()). + AddField(bluge.NewTextField(documentFieldDescription, dashboard.info.Description).SearchTermPositions()) + + // Add legacy ID (for lookup by internal ID) + doc.AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dashboard.id))) + + for _, tag := range dashboard.info.Tags { + doc.AddField(bluge.NewKeywordField(documentFieldTag, tag). + StoreValue(). + Aggregatable(). + SearchTermPositions()) + } + + for _, ds := range dashboard.info.Datasource { + if ds.UID != "" { + doc.AddField(bluge.NewKeywordField(documentFieldDSUID, ds.UID). + StoreValue(). + Aggregatable(). + SearchTermPositions()) + } + if ds.Type != "" { + doc.AddField(bluge.NewKeywordField(documentFieldDSType, ds.Type). + StoreValue(). + Aggregatable(). + SearchTermPositions()) + } + } + + // TODO: enterprise, add dashboard sorting fields + + batch.Insert(doc) + + location += "/" + dashboard.uid + + // Now add a doc for each panel + for _, panel := range dashboard.info.Panels { + uid := dashboard.uid + "#" + strconv.FormatInt(panel.ID, 10) + purl := url + if panel.Type != "row" { + purl = fmt.Sprintf("%s?viewPanel=%d", url, panel.ID) + } + + doc := bluge.NewDocument(uid). + AddField(bluge.NewKeywordField(documentFieldURL, purl).StoreValue()). + AddField(bluge.NewTextField(documentFieldName, panel.Title).StoreValue().SearchTermPositions()). + AddField(bluge.NewTextField(documentFieldDescription, panel.Description).SearchTermPositions()). + AddField(bluge.NewKeywordField(documentFieldPanelType, panel.Type).Aggregatable().StoreValue()). + AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()). + AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindPanel)).Aggregatable().StoreValue()) // likely want independent index for this + + batch.Insert(doc) + } + } + + logger.Info("Inserting documents into bluge batch", "elapsed", time.Since(label)) + label = time.Now() + + err = writer.Batch(batch) + if err != nil { + return nil, err + } + + reader, err := writer.Reader() + if err != nil { + return nil, err + } + + logger.Info("Inserting batch into bluge writer", "elapsed", time.Since(label)) + logger.Info("Finish building bluge index", "totalElapsed", time.Since(start)) + return reader, err +} + +//nolint: gocyclo +func doBlugeQuery(ctx context.Context, s *StandardSearchService, reader *bluge.Reader, filter ResourceFilter, q DashboardQuery) *backend.DataResponse { + response := &backend.DataResponse{} + + // Folder listing structure + idx := strings.Index(q.Query, ":") + if idx > 0 { + key := q.Query[0:idx] + val := q.Query[idx+1:] + if key == "list" { + q.Limit = 1000 + q.Query = "" + q.Location = "" + q.Explain = false + q.SkipLocation = true + q.Facet = nil + if val == "root" || val == "" { + q.Kind = []string{string(entityKindFolder)} + } else { + q.Location = val + q.Kind = []string{string(entityKindDashboard)} + } + } + } + + hasConstraints := false + fullQuery := bluge.NewBooleanQuery() + fullQuery.AddMust(newPermissionFilter(filter, s.logger)) + + // Only show dashboard / folders + if len(q.Kind) > 0 { + bq := bluge.NewBooleanQuery() + for _, k := range q.Kind { + bq.AddShould(bluge.NewTermQuery(k).SetField(documentFieldKind)) + } + fullQuery.AddMust(bq) + hasConstraints = true + } + + // Explicit UID lookup (stars etc) + if len(q.UIDs) > 0 { + bq := bluge.NewBooleanQuery() + for _, v := range q.UIDs { + bq.AddShould(bluge.NewTermQuery(v).SetField(documentFieldUID)) + } + fullQuery.AddMust(bq) + hasConstraints = true + } + + // Legacy lookup by internal ID + if len(q.IDs) > 0 { + bq := bluge.NewBooleanQuery() + for _, v := range q.IDs { + bq.AddShould(bluge.NewTermQuery(fmt.Sprintf("%d", v)).SetField(documentFieldInternalID)) + } + fullQuery.AddMust(bq) + hasConstraints = true + } + + // Tags + if len(q.Tags) > 0 { + bq := bluge.NewBooleanQuery() + for _, v := range q.Tags { + bq.AddMust(bluge.NewTermQuery(v).SetField(documentFieldTag)) + } + fullQuery.AddMust(bq) + hasConstraints = true + } + + // Datasource + if q.Datasource != "" { + fullQuery.AddMust(bluge.NewTermQuery(q.Datasource).SetField(documentFieldDSUID)) + hasConstraints = true + } + + // Folder + if q.Location != "" { + fullQuery.AddMust(bluge.NewTermQuery(q.Location).SetField(documentFieldLocation)) + hasConstraints = true + } + + if q.Query == "*" || q.Query == "" { + if !hasConstraints { + fullQuery.AddShould(bluge.NewMatchAllQuery()) + } + } else { + // The actual se + bq := bluge.NewBooleanQuery(). + AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField("name").SetBoost(6)). + AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField("description").SetBoost(3)). + AddShould(bluge.NewPrefixQuery(q.Query).SetField("name").SetBoost(1)) + + if len(q.Query) > 4 { + bq.AddShould(bluge.NewFuzzyQuery(q.Query).SetField("name")).SetBoost(1.5) + } + fullQuery.AddMust(bq) + } + + limit := 50 // default view + if q.Limit > 0 { + limit = q.Limit + } + + req := bluge.NewTopNSearch(limit, fullQuery) + if q.From > 0 { + req.SetFrom(q.From) + } + if q.Explain { + req.ExplainScores() + } + req.WithStandardAggregations() + + // SortBy([]string{"-_score", "name"}) + // req.SortBy([]string{documentFieldName}) + + for _, t := range q.Facet { + lim := t.Limit + if lim < 1 { + lim = 50 + } + req.AddAggregation(t.Field, aggregations.NewTermsAggregation(search.Field(t.Field), lim)) + } + + // execute this search on the reader + documentMatchIterator, err := reader.Search(ctx, req) + if err != nil { + s.logger.Error("error executing search: %v", err) + response.Error = err + return response + } + + dvfieldNames := []string{"type"} + sctx := search.NewSearchContext(0, 0) + + // numericFields := map[string]bool{"schemaVersion": true, "panelCount": true} + + fScore := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0) + fUID := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fKind := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fPType := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fName := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fURL := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fLocation := data.NewFieldFromFieldType(data.FieldTypeString, 0) + fTags := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0) + fDSUIDs := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0) + fExplain := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0) + + fScore.Name = "score" + fUID.Name = "uid" + fKind.Name = "kind" + fName.Name = "name" + fLocation.Name = "location" + fURL.Name = "url" + fURL.Config = &data.FieldConfig{ + Links: []data.DataLink{ + {Title: "link", URL: "${__value.text}"}, + }, + } + fPType.Name = "panel_type" + fDSUIDs.Name = "ds_uid" + fTags.Name = "tags" + fExplain.Name = "explain" + + frame := data.NewFrame("Query results", fScore, fKind, fUID, fName, fPType, fURL, fTags, fDSUIDs, fLocation) + if q.Explain { + frame.Fields = append(frame.Fields, fExplain) + } + + locationItems := make(map[string]bool, 50) + + // iterate through the document matches + match, err := documentMatchIterator.Next() + for err == nil && match != nil { + err = match.LoadDocumentValues(sctx, dvfieldNames) + if err != nil { + continue + } + + uid := "" + kind := "" + ptype := "" + name := "" + url := "" + loc := "" + var ds_uids []string + var tags []string + + err = match.VisitStoredFields(func(field string, value []byte) bool { + // if numericFields[field] { + // num, err2 := bluge.DecodeNumericFloat64(value) + // if err2 != nil { + // vals[field] = num + // } + // } else { + // vals[field] = string(value) + // } + + switch field { + case documentFieldUID: + uid = string(value) + case documentFieldKind: + kind = string(value) + case documentFieldPanelType: + ptype = string(value) + case documentFieldName: + name = string(value) + case documentFieldURL: + url = string(value) + case documentFieldLocation: + loc = string(value) + case documentFieldDSUID: + ds_uids = append(ds_uids, string(value)) + case documentFieldTag: + tags = append(tags, string(value)) + } + return true + }) + if err != nil { + s.logger.Error("error loading stored fields: %v", err) + response.Error = err + return response + } + + fScore.Append(match.Score) + fKind.Append(kind) + fUID.Append(uid) + fPType.Append(ptype) + fName.Append(name) + fURL.Append(url) + fLocation.Append(loc) + + // set a key for all path parts we return + if !q.SkipLocation { + for _, v := range strings.Split(loc, "/") { + locationItems[v] = true + } + } + + if len(tags) > 0 { + js, _ := json.Marshal(tags) + jsb := json.RawMessage(js) + fTags.Append(&jsb) + } else { + fTags.Append(nil) + } + + if len(ds_uids) > 0 { + js, _ := json.Marshal(ds_uids) + jsb := json.RawMessage(js) + fDSUIDs.Append(&jsb) + } else { + fDSUIDs.Append(nil) + } + + if q.Explain { + if match.Explanation != nil { + js, _ := json.Marshal(&match.Explanation) + jsb := json.RawMessage(js) + fExplain.Append(&jsb) + } else { + fExplain.Append(nil) + } + } + + // load the next document match + match, err = documentMatchIterator.Next() + } + + // Must call after iterating :) + aggs := documentMatchIterator.Aggregations() + + header := &customMeta{ + Count: aggs.Count(), // Total cound + MaxScore: aggs.Metric("max_score"), + } + if len(locationItems) > 0 && !q.SkipLocation { + header.Locations = getLocationLookupInfo(ctx, reader, locationItems) + } + + frame.SetMeta(&data.FrameMeta{ + Type: "search-results", + Custom: header, + }) + + response.Frames = append(response.Frames, frame) + + for _, t := range q.Facet { + bbb := aggs.Buckets(t.Field) + if bbb != nil { + size := len(bbb) + + fName := data.NewFieldFromFieldType(data.FieldTypeString, size) + fName.Name = t.Field + + fCount := data.NewFieldFromFieldType(data.FieldTypeUint64, size) + fCount.Name = "Count" + + for i, v := range bbb { + fName.Set(i, v.Name()) + fCount.Set(i, v.Count()) + } + + response.Frames = append(response.Frames, data.NewFrame("Facet: "+t.Field, fName, fCount)) + } + } + + return response +} + +func getLocationLookupInfo(ctx context.Context, reader *bluge.Reader, uids map[string]bool) map[string]locationItem { + res := make(map[string]locationItem, len(uids)) + bq := bluge.NewBooleanQuery() + for k := range uids { + bq.AddShould(bluge.NewTermQuery(k).SetField(documentFieldUID)) + } + + req := bluge.NewAllMatches(bq) + + documentMatchIterator, err := reader.Search(ctx, req) + if err != nil { + return res + } + + dvfieldNames := []string{"type"} + sctx := search.NewSearchContext(0, 0) + + // execute this search on the reader + // iterate through the document matches + match, err := documentMatchIterator.Next() + for err == nil && match != nil { + err = match.LoadDocumentValues(sctx, dvfieldNames) + if err != nil { + continue + } + + uid := "" + item := locationItem{} + + _ = match.VisitStoredFields(func(field string, value []byte) bool { + switch field { + case documentFieldUID: + uid = string(value) + case documentFieldKind: + item.Kind = string(value) + case documentFieldName: + item.Name = string(value) + case documentFieldURL: + item.URL = string(value) + } + return true + }) + + res[uid] = item + + // load the next document match + match, err = documentMatchIterator.Next() + } + + return res +} + +type locationItem struct { + Name string `json:"name"` + Kind string `json:"kind"` + URL string `json:"url"` +} + +type customMeta struct { + Count uint64 `json:"count"` + MaxScore float64 `json:"max_score,omitempty"` + Locations map[string]locationItem `json:"locationInfo,omitempty"` +} diff --git a/pkg/services/searchV2/filter.go b/pkg/services/searchV2/filter.go new file mode 100644 index 00000000000..354e7fdc499 --- /dev/null +++ b/pkg/services/searchV2/filter.go @@ -0,0 +1,132 @@ +package searchV2 + +import ( + "regexp" + + "github.com/blugelabs/bluge" + "github.com/blugelabs/bluge/search" + "github.com/blugelabs/bluge/search/searcher" + "github.com/blugelabs/bluge/search/similarity" + "github.com/grafana/grafana/pkg/infra/log" +) + +type PermissionFilter struct { + log log.Logger + filter ResourceFilter +} + +type entityKind string + +const ( + entityKindPanel entityKind = "panel" + entityKindDashboard entityKind = "dashboard" + entityKindFolder entityKind = "folder" +) + +func (r entityKind) IsValid() bool { + return r == entityKindPanel || r == entityKindDashboard || r == entityKindFolder +} + +func (r entityKind) supportsAuthzCheck() bool { + return r == entityKindPanel || r == entityKindDashboard || r == entityKindFolder +} + +var ( + permissionFilterFields = []string{documentFieldUID, documentFieldKind} + panelIdFieldRegex = regexp.MustCompile(`^(.*)#([0-9]{1,4})$`) + panelIdFieldDashboardUidSubmatchIndex = 1 + panelIdFieldPanelIdSubmatchIndex = 2 + panelIdFieldRegexExpectedSubmatchCount = 3 // submatches[0] - whole string + + _ bluge.Query = (*PermissionFilter)(nil) +) + +func newPermissionFilter(resourceFilter ResourceFilter, log log.Logger) *PermissionFilter { + return &PermissionFilter{ + filter: resourceFilter, + log: log, + } +} + +func (q *PermissionFilter) logAccessDecision(decision bool, kind interface{}, id string, reason string, ctx ...interface{}) { + if true { + return // TOO much logging right now + } + + ctx = append(ctx, "kind", kind, "id", id, "reason", reason) + if decision { + q.log.Debug("allowing access", ctx...) + } else { + q.log.Info("denying access", ctx...) + } +} + +func (q *PermissionFilter) canAccess(kind entityKind, id string) bool { + if !kind.supportsAuthzCheck() { + q.logAccessDecision(false, kind, id, "entityDoesNotSupportAuthz") + return false + } + + // TODO add `kind` to the `ResourceFilter` interface so that we can move the switch out of here + // + switch kind { + case entityKindFolder: + if id == "" { + q.logAccessDecision(true, kind, id, "generalFolder") + return true + } + fallthrough + case entityKindDashboard: + decision := q.filter(id) + q.logAccessDecision(decision, kind, id, "resourceFilter") + return decision + case entityKindPanel: + matches := panelIdFieldRegex.FindStringSubmatch(id) + + submatchCount := len(matches) + if submatchCount != panelIdFieldRegexExpectedSubmatchCount { + q.logAccessDecision(false, kind, id, "invalidPanelIdFieldRegexSubmatchCount", "submatchCount", submatchCount, "expectedSubmatchCount", panelIdFieldRegexExpectedSubmatchCount) + return false + } + + dashboardUid := matches[panelIdFieldDashboardUidSubmatchIndex] + decision := q.filter(dashboardUid) + + q.logAccessDecision(decision, kind, id, "resourceFilter", "dashboardUid", dashboardUid, "panelId", matches[panelIdFieldPanelIdSubmatchIndex]) + return decision + default: + q.logAccessDecision(false, kind, id, "reason", "unknownKind") + return false + } +} + +func (q *PermissionFilter) Searcher(i search.Reader, options search.SearcherOptions) (search.Searcher, error) { + dvReader, err := i.DocumentValueReader(permissionFilterFields) + if err != nil { + return nil, err + } + + s, err := searcher.NewMatchAllSearcher(i, 1, similarity.ConstantScorer(1), options) + return searcher.NewFilteringSearcher(s, func(d *search.DocumentMatch) bool { + var kind, id string + err := dvReader.VisitDocumentValues(d.Number, func(field string, term []byte) { + if field == documentFieldKind { + kind = string(term) + } else if field == documentFieldUID { + id = string(term) + } + }) + if err != nil { + q.logAccessDecision(false, kind, id, "errorWhenVisitingDocumentValues") + return false + } + + e := entityKind(kind) + if !e.IsValid() { + q.logAccessDecision(false, kind, id, "invalidEntityKind") + return false + } + + return q.canAccess(e, id) + }), err +} diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index 79dbed598ed..c6ddeecc07b 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -13,6 +13,8 @@ import ( "github.com/grafana/grafana/pkg/services/searchV2/extract" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/store" + + "github.com/blugelabs/bluge" ) type dashboardLoader interface { @@ -32,7 +34,8 @@ type eventStore interface { type dashboardIndex struct { mu sync.RWMutex loader dashboardLoader - dashboards map[int64][]dashboard // orgId -> []dashboards + dashboards map[int64][]dashboard // orgId -> []dashboards + reader map[int64]*bluge.Reader // orgId -> bluge index eventStore eventStore logger log.Logger } @@ -53,6 +56,7 @@ func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore) *dashboar loader: dashLoader, eventStore: evStore, dashboards: map[int64][]dashboard{}, + reader: map[int64]*bluge.Reader{}, logger: log.New("dashboardIndex"), } } @@ -81,6 +85,9 @@ func (i *dashboardIndex) run(ctx context.Context) error { } i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", len(dashboards)) + // build bluge index on startup (will catch panics) + go i.reIndexFromScratchBluge(ctx) + for { select { case <-partialUpdateTicker.C: @@ -120,6 +127,60 @@ func (i *dashboardIndex) reIndexFromScratch(ctx context.Context) { } } +// Variation of the above function that builds bluge index from scratch +// Once the frontend is wired up, we should switch to this one +func (i *dashboardIndex) reIndexFromScratchBluge(ctx context.Context) { + // Catch Panic (just in case) + defer func() { + recv := recover() + if recv != nil { + i.logger.Error("panic in search runner", "recv", recv) // REMVOE after we are sure it works! + } + }() + + i.mu.RLock() + orgIDs := make([]int64, 0, len(i.dashboards)) + for orgID := range i.dashboards { + orgIDs = append(orgIDs, orgID) + } + i.mu.RUnlock() + if len(orgIDs) < 1 { + orgIDs = append(orgIDs, int64(1)) // make sure we index + } + + for _, orgID := range orgIDs { + started := time.Now() + ctx, cancel := context.WithTimeout(ctx, time.Minute) + + dashboards, err := i.loader.LoadDashboards(ctx, orgID, "") + if err != nil { + cancel() + i.logger.Error("Error re-indexing dashboards for organization", "orgId", orgID, "error", err) + continue + } + orgSearchIndexLoadTime := time.Since(started) + + reader, err := initBlugeIndex(dashboards, i.logger) + if err != nil { + cancel() + i.logger.Error("Error re-indexing dashboards for organization", "orgId", orgID, "error", err) + continue + } + orgSearchIndexTotalTime := time.Since(started) + orgSearchIndexBuildTime := orgSearchIndexTotalTime - orgSearchIndexLoadTime + + cancel() + i.logger.Info("Re-indexed dashboards for organization (bluge)", + "orgId", orgID, + "orgSearchIndexLoadTime", orgSearchIndexLoadTime, + "orgSearchIndexBuildTime", orgSearchIndexBuildTime, + "orgSearchIndexTotalTime", orgSearchIndexTotalTime) + i.mu.Lock() + i.reader[orgID] = reader + i.mu.Unlock() + } +} + func (i *dashboardIndex) applyIndexUpdates(ctx context.Context, lastEventID int64) int64 { events, err := i.eventStore.GetAllEventsAfter(context.Background(), lastEventID) if err != nil { diff --git a/pkg/services/searchV2/service.go b/pkg/services/searchV2/service.go index 17a098edd45..773318d45bb 100644 --- a/pkg/services/searchV2/service.go +++ b/pkg/services/searchV2/service.go @@ -105,8 +105,24 @@ func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backen return user, nil } -func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, _ DashboardQuery) *backend.DataResponse { +func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, q DashboardQuery) *backend.DataResponse { rsp := &backend.DataResponse{} + signedInUser, err := s.getUser(ctx, user, orgId) + if err != nil { + rsp.Error = err + return rsp + } + + filter, err := s.auth.GetDashboardReadFilter(signedInUser) + if err != nil { + rsp.Error = err + return rsp + } + + reader := s.dashboardIndex.reader[orgId] + if reader != nil && q.Query != "" { // frontend initializes with empty string + return doBlugeQuery(ctx, s, reader, filter, q) + } dashboards, err := s.dashboardIndex.getDashboards(ctx, orgId) if err != nil { @@ -114,29 +130,14 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back return rsp } - signedInUser, err := s.getUser(ctx, user, orgId) - if err != nil { - rsp.Error = err - return rsp - } - - dashboards, err = s.applyAuthFilter(signedInUser, dashboards) - if err != nil { - rsp.Error = err - return rsp - } + dashboards = s.applyAuthFilter(filter, dashboards) rsp.Frames = metaToFrame(dashboards) return rsp } -func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashboards []dashboard) ([]dashboard, error) { - filter, err := s.auth.GetDashboardReadFilter(user) - if err != nil { - return nil, err - } - +func (s *StandardSearchService) applyAuthFilter(filter ResourceFilter, dashboards []dashboard) []dashboard { // create a list of all viewable dashboards for this user. res := make([]dashboard, 0, len(dashboards)) for _, dash := range dashboards { @@ -144,7 +145,7 @@ func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashb res = append(res, dash) } } - return res, nil + return res } type simpleCounter struct { diff --git a/pkg/services/searchV2/types.go b/pkg/services/searchV2/types.go index 12fddfb6d3e..33475bd7398 100644 --- a/pkg/services/searchV2/types.go +++ b/pkg/services/searchV2/types.go @@ -8,8 +8,27 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" ) +type FacetField struct { + Field string `json:"field"` + Limit int `json:"limit,omitempty"` // explicit page size +} + type DashboardQuery struct { - Query string + Query string `json:"query"` + Location string `json:"location,omitempty"` // parent folder ID + Sort string `json:"sort,omitempty"` // field ASC/DESC + Datasource string `json:"ds_uid,omitempty"` // "datasource" collides with the JSON value at the same leel :() + Tags []string `json:"tags,omitempty"` + Kind []string `json:"kind,omitempty"` + UIDs []string `json:"uid,omitempty"` + IDs []int64 `json:"id,omitempty"` // deprecated -- but will convert internal ID to UIDs + Explain bool `json:"explain,omitempty"` // adds details on why document matched + Facet []FacetField `json:"facet,omitempty"` + SkipLocation bool `json:"skipLocation,omitempty"` + AccessInfo bool `json:"accessInfo,omitempty"` // adds field for access control + HasPreview string `json:"hasPreview,omitempty"` // the light|dark theme + Limit int `json:"limit,omitempty"` // explicit page size + From int `json:"from,omitempty"` // for paging } type SearchService interface {