Search: Add substring matcher (#54895)

* Search: Add substring matcher (#54813)

* search: bring back substring search

* lint fix

* search: bring back ngram

* search: bring back ngram

* search: more tests

* fix tests
This commit is contained in:
Artur Wierzbicki
2022-09-09 14:20:20 +02:00
committed by GitHub
parent 19a58da54c
commit e1f5d698e6
11 changed files with 662 additions and 74 deletions
+29 -16
View File
@@ -24,7 +24,6 @@ const (
documentFieldName = "name"
documentFieldName_sort = "name_sort"
documentFieldName_ngram = "name_ngram"
documentFieldDescription = "description"
documentFieldLocation = "location" // parent path
documentFieldPanelType = "panel_type"
documentFieldTransformer = "transformer"
@@ -232,14 +231,11 @@ func newSearchDocument(uid string, name string, descr string, url string) *bluge
doc.AddField(bluge.NewTextField(documentFieldName_ngram, name).WithAnalyzer(ngramIndexAnalyzer))
// Don't add a field for empty names
sortStr := strings.Trim(strings.ToUpper(name), " ")
sortStr := formatForNameSortField(name)
if len(sortStr) > 0 {
doc.AddField(bluge.NewKeywordField(documentFieldName_sort, sortStr).Sortable())
}
}
if descr != "" {
doc.AddField(bluge.NewTextField(documentFieldDescription, descr).SearchTermPositions())
}
if url != "" {
doc.AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue())
}
@@ -435,21 +431,19 @@ func doSearchQuery(
fullQuery.AddShould(bluge.NewMatchAllQuery())
}
} else {
// The actual se
bq := bluge.NewBooleanQuery().
AddShould(bluge.NewMatchQuery(q.Query).SetField(documentFieldName).SetBoost(6)).
AddShould(bluge.NewMatchQuery(q.Query).SetField(documentFieldDescription).SetBoost(3)).
AddShould(bluge.NewMatchQuery(q.Query).
bq := bluge.NewBooleanQuery()
bq.AddShould(NewSubstringQuery(formatForNameSortField(q.Query)).
SetField(documentFieldName_sort).
SetBoost(6))
if shouldUseNgram(q) {
bq.AddShould(bluge.NewMatchQuery(q.Query).
SetField(documentFieldName_ngram).
SetOperator(bluge.MatchQueryOperatorAnd). // all terms must match
SetAnalyzer(ngramQueryAnalyzer).SetBoost(1))
}
if len(q.Query) > 4 {
bq.AddShould(bluge.NewFuzzyQuery(q.Query).SetField(documentFieldName)).SetBoost(1.5)
}
if len(q.Query) > ngramEdgeFilterMaxLength && !strings.Contains(q.Query, " ") {
bq.AddShould(bluge.NewPrefixQuery(strings.ToLower(q.Query)).SetField(documentFieldName)).SetBoost(6)
}
fullQuery.AddMust(bq)
}
@@ -664,6 +658,25 @@ func doSearchQuery(
return response
}
func shouldUseNgram(q DashboardQuery) bool {
var tokens []string
if len(q.Query) > ngramEdgeFilterMaxLength {
tokens = strings.Fields(q.Query)
for _, k := range tokens {
// ngram will never match if at least one input token exceeds the max token length,
// as all tokens must match simultaneously with the `bluge.MatchQueryOperatorAnd` operator
if len(k) > ngramEdgeFilterMaxLength {
return false
}
}
}
return true
}
func formatForNameSortField(name string) string {
return strings.Trim(strings.ToUpper(name), " ")
}
func getLocationLookupInfo(ctx context.Context, reader *bluge.Reader, uids map[string]bool) map[string]locationItem {
res := make(map[string]locationItem, len(uids))
bq := bluge.NewBooleanQuery()
+160
View File
@@ -6,6 +6,7 @@ import (
"path/filepath"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
@@ -82,6 +83,33 @@ func checkSearchResponseExtended(t *testing.T, fileName string, index *orgIndex,
experimental.CheckGoldenJSONResponse(t, "testdata", fileName, resp, true)
}
func getFrameWithNames(resp *backend.DataResponse) *data.Frame {
if resp == nil || len(resp.Frames) == 0 {
return nil
}
frame := resp.Frames[0]
nameField, idx := frame.FieldByName(documentFieldName)
if nameField.Len() == 0 || idx == -1 {
return nil
}
scoreField, _ := frame.FieldByName("score")
return data.NewFrame("ordering frame", nameField, scoreField)
}
func checkSearchResponseOrdering(t *testing.T, fileName string, index *orgIndex, filter ResourceFilter, query DashboardQuery) {
t.Helper()
checkSearchResponseOrderingExtended(t, fileName, index, filter, query, &NoopQueryExtender{})
}
func checkSearchResponseOrderingExtended(t *testing.T, fileName string, index *orgIndex, filter ResourceFilter, query DashboardQuery, extender QueryExtender) {
t.Helper()
query.Explain = true
resp := doSearchQuery(context.Background(), testLogger, index, filter, query, extender, "/pfix")
experimental.CheckGoldenJSONFrame(t, "testdata", fileName, getFrameWithNames(resp), true)
}
var testDashboards = []dashboard{
{
id: 1,
@@ -582,3 +610,135 @@ func TestDashboardIndex_CamelCaseNgram(t *testing.T) {
)
})
}
func dashboardsWithTitles(names ...string) []dashboard {
out := make([]dashboard, 0)
for i, name := range names {
no := int64(i + 1)
out = append(out, dashboard{
id: no,
uid: fmt.Sprintf("%d", no),
info: &extract.DashboardInfo{
Title: name,
},
})
}
return out
}
func TestDashboardIndex_MultiTermPrefixMatch(t *testing.T) {
var tests = []struct {
dashboards []dashboard
query string
}{
{
dashboards: dashboardsWithTitles(
"Panel Tests - Bar Gauge 2",
"Prometheus 2.0",
"Prometheus 2.0 Stats",
"Prometheus 20.0",
"Prometheus Second Word",
"Prometheus Stats",
"dynamic (2)",
"prometheus histogram",
"prometheus histogram2",
"roci-simple-2",
"x not y",
),
query: "Prometheus 2.",
},
{
dashboards: dashboardsWithTitles(
"From AAA",
"Grafana Dev Overview & Home",
"Home automation",
"Prometheus 2.0",
"Prometheus 2.0 Stats",
"Prometheus 20.0",
"Prometheus Stats",
"Transforms - config from query",
"iot-testing",
"prom style with exemplars",
"prop history",
"simple frame",
"with-hide-from",
"xy broke",
),
query: "Prome",
},
{
dashboards: dashboardsWithTitles(
"Panel Tests - Bar Gauge 2",
"Prometheus 2.0",
"Prometheus 2.0 Stats",
"Prometheus 20.0",
"Prometheus Second Word",
"Prometheus Stats",
"dynamic (2)",
"prometheus histogram",
"prometheus histogram2",
"roci-simple-2",
"x not y",
),
query: "Prometheus stat",
},
{
dashboards: dashboardsWithTitles(
"Loki Tests - Bar Gauge 2",
"Loki 2.0",
"Loki 2.0 Stats",
"Loki 20.0",
"Loki Second Word",
"Loki Stats",
"dynamic (2)",
"Loki histogram",
"Loki histogram2",
"roci-simple-2",
"x not y",
),
query: "Loki 2.",
},
{
dashboards: dashboardsWithTitles(
"Loki Tests - Bar Gauge 2",
"Loki 2.0",
"Loki 2.0 Stats",
"Loki 20.0",
"Loki Second Word",
"Loki Stats",
"dynamic (2)",
"Loki histogram",
"Loki histogram2",
"roci-simple-2",
"x not y",
),
query: "Lok",
},
{
dashboards: dashboardsWithTitles(
"Loki Tests - Bar Gauge 2",
"Loki 2.0",
"Loki 2.0 Stats",
"Loki 20.0",
"Loki Second Word",
"Loki Stats",
"dynamic (2)",
"Loki histogram",
"Loki histogram2",
"roci-simple-2",
"x not y",
),
query: "Loki stats",
},
}
for i, tt := range tests {
t.Run(fmt.Sprintf("ordering-tests-%d-[%s]", i+1, tt.query), func(t *testing.T) {
index := initTestOrgIndexFromDashes(t, tt.dashboards)
checkSearchResponseOrdering(t, filepath.Base(t.Name()), index, testAllowAllFilter,
DashboardQuery{Query: tt.query},
)
})
}
}
+102
View File
@@ -0,0 +1,102 @@
// based on https://github.com/blugelabs/bluge/blob/57414197005148539c5dc5db8ab581594969df79/query.go#L1407-L1482, license:
// Copyright (c) 2020 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package searchV2
import (
"strings"
"github.com/blugelabs/bluge/search"
"github.com/blugelabs/bluge/search/searcher"
"github.com/blugelabs/bluge/search/similarity"
)
type boost float64
func (b *boost) Value() float64 {
if b == nil {
return 1
}
return float64(*b)
}
type SubstringQuery struct {
substring string
field string
boost *boost
scorer search.Scorer
}
func NewSubstringQuery(wildcard string) *SubstringQuery {
return &SubstringQuery{
substring: wildcard,
}
}
// Wildcard returns the substring being queried
func (q *SubstringQuery) Wildcard() string {
return q.substring
}
func (q *SubstringQuery) SetBoost(b float64) *SubstringQuery {
boostVal := boost(b)
q.boost = &boostVal
return q
}
func (q *SubstringQuery) Boost() float64 {
return q.boost.Value()
}
func (q *SubstringQuery) SetField(f string) *SubstringQuery {
q.field = f
return q
}
func (q *SubstringQuery) Field() string {
return q.field
}
var regexpEscaper = strings.NewReplacer(
// characters in the substring that must
// be escaped in the regexp
"+", `\+`,
"*", `\*`,
"(", `\(`,
")", `\)`,
"^", `\^`,
"$", `\$`,
".", `\.`,
"{", `\{`,
"}", `\}`,
"[", `\[`,
"]", `\]`,
`|`, `\|`,
`\`, `\\`)
func (q *SubstringQuery) Searcher(i search.Reader, options search.SearcherOptions) (search.Searcher, error) {
field := q.field
if q.field == "" {
field = options.DefaultSearchField
}
regexpString := ".*" + regexpEscaper.Replace(q.substring) + ".*"
return searcher.NewRegexpStringSearcher(i, regexpString, field,
q.boost.Value(), q.scorer, similarity.NewCompositeSumScorer(), options)
}
func (q *SubstringQuery) Validate() error {
return nil // real validation delayed until searcher constructor
}
@@ -0,0 +1,53 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 2 Rows
// +----------------------+-------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +----------------------+-------------------+
// | Prometheus 2.0 | 7.621131552585596 |
// | Prometheus 2.0 Stats | 7.621131552585596 |
// +----------------------+-------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Prometheus 2.0",
"Prometheus 2.0 Stats"
],
[
7.621131552585596,
7.621131552585596
]
]
}
}
]
}
@@ -0,0 +1,59 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 4 Rows
// +----------------------+-------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +----------------------+-------------------+
// | Prometheus 2.0 | 9.502378631081775 |
// | Prometheus 20.0 | 9.458475876581684 |
// | Prometheus Stats | 9.379374926302209 |
// | Prometheus 2.0 Stats | 9.310081326315688 |
// +----------------------+-------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Prometheus 2.0",
"Prometheus 20.0",
"Prometheus Stats",
"Prometheus 2.0 Stats"
],
[
9.502378631081775,
9.458475876581684,
9.379374926302209,
9.310081326315688
]
]
}
}
]
}
@@ -0,0 +1,50 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 1 Rows
// +------------------+-------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +------------------+-------------------+
// | Prometheus Stats | 7.621131552585596 |
// +------------------+-------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Prometheus Stats"
],
[
7.621131552585596
]
]
}
}
]
}
@@ -0,0 +1,62 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 5 Rows
// +--------------------------+--------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +--------------------------+--------------------+
// | Loki 2.0 | 9.273036652923247 |
// | Loki 2.0 Stats | 8.951742733604135 |
// | Loki 20.0 | 2.57580523764178 |
// | Loki histogram2 | 2.2807887502617943 |
// | Loki Tests - Bar Gauge 2 | 2.045832444623899 |
// +--------------------------+--------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Loki 2.0",
"Loki 2.0 Stats",
"Loki 20.0",
"Loki histogram2",
"Loki Tests - Bar Gauge 2"
],
[
9.273036652923247,
8.951742733604135,
2.57580523764178,
2.2807887502617943,
2.045832444623899
]
]
}
}
]
}
@@ -0,0 +1,71 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 8 Rows
// +--------------------------+-------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +--------------------------+-------------------+
// | Loki 2.0 | 8.386497572003142 |
// | Loki 20.0 | 8.351238737232393 |
// | Loki Stats | 8.289644692681875 |
// | Loki 2.0 Stats | 8.237634633059454 |
// | Loki histogram | 8.237634633059454 |
// | Loki histogram2 | 8.214550743132483 |
// | Loki Second Word | 8.173207674966303 |
// | Loki Tests - Bar Gauge 2 | 8.105690026892566 |
// +--------------------------+-------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Loki 2.0",
"Loki 20.0",
"Loki Stats",
"Loki 2.0 Stats",
"Loki histogram",
"Loki histogram2",
"Loki Second Word",
"Loki Tests - Bar Gauge 2"
],
[
8.386497572003142,
8.351238737232393,
8.289644692681875,
8.237634633059454,
8.237634633059454,
8.214550743132483,
8.173207674966303,
8.105690026892566
]
]
}
}
]
}
@@ -0,0 +1,53 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0]
// Name: ordering frame
// Dimensions: 2 Fields by 2 Rows
// +----------------+-------------------+
// | Name: name | Name: score |
// | Labels: | Labels: |
// | Type: []string | Type: []float64 |
// +----------------+-------------------+
// | Loki Stats | 9.397899591158676 |
// | Loki 2.0 Stats | 2.638536183312778 |
// +----------------+-------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "ordering frame",
"fields": [
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "score",
"type": "number",
"typeInfo": {
"frame": "float64"
}
}
]
},
"data": {
"values": [
[
"Loki Stats",
"Loki 2.0 Stats"
],
[
9.397899591158676,
2.638536183312778
]
]
}
}
]
}
@@ -3,19 +3,17 @@
// Frame[0] {
// "type": "search-results",
// "custom": {
// "count": 2
// "count": 0
// }
// }
// Name: Query results
// Dimensions: 8 Fields by 2 Rows
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []json.RawMessage | Type: []string |
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
// | dashboard | 2 | A secret is powerful when it is empty (Umberto Eco) | | /pfix/d/2/ | null | [] | |
// | dashboard | 1 | Three can keep a secret, if two of them are dead (Benjamin Franklin) | | /pfix/d/1/ | null | [] | |
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
// Dimensions: 8 Fields by 0 Rows
// +----------------+----------------+----------------+------------------+----------------+--------------------------+-------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []json.RawMessage | Type: []string |
// +----------------+----------------+----------------+------------------+----------------+--------------------------+-------------------------+----------------+
// +----------------+----------------+----------------+------------------+----------------+--------------------------+-------------------------+----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
@@ -27,7 +25,7 @@
"meta": {
"type": "search-results",
"custom": {
"count": 2
"count": 0
}
},
"fields": [
@@ -100,38 +98,14 @@
},
"data": {
"values": [
[
"dashboard",
"dashboard"
],
[
"2",
"1"
],
[
"A secret is powerful when it is empty (Umberto Eco)",
"Three can keep a secret, if two of them are dead (Benjamin Franklin)"
],
[
"",
""
],
[
"/pfix/d/2/",
"/pfix/d/1/"
],
[
null,
null
],
[
[],
[]
],
[
"",
""
]
[],
[],
[],
[],
[],
[],
[],
[]
]
}
}
+6 -15
View File
@@ -3,18 +3,17 @@
// Frame[0] {
// "type": "search-results",
// "custom": {
// "count": 2
// "count": 1
// }
// }
// Name: Query results
// Dimensions: 8 Fields by 2 Rows
// Dimensions: 8 Fields by 1 Rows
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []json.RawMessage | Type: []string |
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
// | dashboard | 1 | Three can keep a secret, if two of them are dead (Benjamin Franklin) | | /pfix/d/1/ | null | [] | |
// | dashboard | 2 | A secret is powerful when it is empty (Umberto Eco) | | /pfix/d/2/ | null | [] | |
// +----------------+----------------+----------------------------------------------------------------------+------------------+----------------+--------------------------+-------------------------+----------------+
//
//
@@ -27,7 +26,7 @@
"meta": {
"type": "search-results",
"custom": {
"count": 2
"count": 1
}
},
"fields": [
@@ -101,35 +100,27 @@
"data": {
"values": [
[
"dashboard",
"dashboard"
],
[
"1",
"2"
"1"
],
[
"Three can keep a secret, if two of them are dead (Benjamin Franklin)",
"A secret is powerful when it is empty (Umberto Eco)"
"Three can keep a secret, if two of them are dead (Benjamin Franklin)"
],
[
"",
""
],
[
"/pfix/d/1/",
"/pfix/d/2/"
"/pfix/d/1/"
],
[
null,
null
],
[
[],
[]
],
[
"",
""
]
]