From 7411c81e7f029414b31e50ede3d755d86f06c429 Mon Sep 17 00:00:00 2001 From: Alexander Emelin Date: Tue, 24 May 2022 00:33:56 +0300 Subject: [PATCH] Search: use prefix search for queries exceeding ngram filter max length (#49409) --- pkg/services/searchV2/bluge.go | 7 ++++++- pkg/services/searchV2/index_test.go | 19 +++++++++++++++++ .../testdata/prefix-search-ngram-exceeded.txt | 21 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 pkg/services/searchV2/testdata/prefix-search-ngram-exceeded.txt diff --git a/pkg/services/searchV2/bluge.go b/pkg/services/searchV2/bluge.go index 5e9dbde24e2..d4c16ea1761 100644 --- a/pkg/services/searchV2/bluge.go +++ b/pkg/services/searchV2/bluge.go @@ -222,11 +222,13 @@ func getDashboardPanelDocs(dash dashboard, location string) []*bluge.Document { return docs } +const ngramEdgeFilterMaxLength = 7 + var ngramIndexAnalyzer = &analysis.Analyzer{ Tokenizer: tokenizer.NewWhitespaceTokenizer(), TokenFilters: []analysis.TokenFilter{ token.NewLowerCaseFilter(), - token.NewEdgeNgramFilter(token.FRONT, 1, 7), + token.NewEdgeNgramFilter(token.FRONT, 1, ngramEdgeFilterMaxLength), }, } @@ -379,6 +381,9 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader, if len(q.Query) > 4 { bq.AddShould(bluge.NewFuzzyQuery(q.Query).SetField(documentFieldName)).SetBoost(1.5) } + if len(q.Query) > ngramEdgeFilterMaxLength && !strings.Contains(q.Query, " ") { + bq.AddShould(bluge.NewPrefixQuery(strings.ToLower(q.Query)).SetField(documentFieldName)).SetBoost(6) + } fullQuery.AddMust(bq) } diff --git a/pkg/services/searchV2/index_test.go b/pkg/services/searchV2/index_test.go index 9a6d8b7bc8c..662ae09e26b 100644 --- a/pkg/services/searchV2/index_test.go +++ b/pkg/services/searchV2/index_test.go @@ -322,6 +322,25 @@ func TestDashboardIndex_MultipleTokensInRow(t *testing.T) { }) } +var longPrefixDashboards = []dashboard{ + { + id: 1, + uid: "1", + info: &extract.DashboardInfo{ + Title: "Eyjafjallajökull Eruption data", + }, + }, +} + +func TestDashboardIndex_PrefixNgramExceeded(t *testing.T) { + t.Run("prefix-search-ngram-exceeded", func(t *testing.T) { + _, reader, _ := initTestIndexFromDashes(t, longPrefixDashboards) + checkSearchResponse(t, filepath.Base(t.Name())+".txt", reader, testAllowAllFilter, + DashboardQuery{Query: "Eyjafjallajöku"}, + ) + }) +} + var scatteredTokensDashboards = []dashboard{ { id: 1, diff --git a/pkg/services/searchV2/testdata/prefix-search-ngram-exceeded.txt b/pkg/services/searchV2/testdata/prefix-search-ngram-exceeded.txt new file mode 100644 index 00000000000..c6c870d0b3d --- /dev/null +++ b/pkg/services/searchV2/testdata/prefix-search-ngram-exceeded.txt @@ -0,0 +1,21 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "type": "search-results", + "custom": { + "count": 1 + } +} +Name: Query results +Dimensions: 8 Fields by 1 Rows ++----------------+----------------+--------------------------------+------------------+----------------+--------------------------+--------------------------+----------------+ +| Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location | +| Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | +| Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []string | ++----------------+----------------+--------------------------------+------------------+----------------+--------------------------+--------------------------+----------------+ +| dashboard | 1 | Eyjafjallajökull Eruption data | | /d/1/ | null | null | | ++----------------+----------------+--------------------------------+------------------+----------------+--------------------------+--------------------------+----------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////UAQAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAKwAAAADAAAAWAAAACgAAAAEAAAAPPz//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABc/P//CAAAABgAAAANAAAAUXVlcnkgcmVzdWx0cwAAAAQAAABuYW1lAAAAAIj8//8IAAAAOAAAAC4AAAB7InR5cGUiOiJzZWFyY2gtcmVzdWx0cyIsImN1c3RvbSI6eyJjb3VudCI6MX19AAAEAAAAbWV0YQAAAAAIAAAACAMAAKACAABEAgAA4AEAADQBAADYAAAAaAAAAAQAAAAq/f//FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAABj9//8IAAAAFAAAAAgAAABsb2NhdGlvbgAAAAAEAAAAbmFtZQAAAAAAAAAAFP3//wgAAABsb2NhdGlvbgAAAACm////FAAAADwAAAA8AAAAAAAEATgAAAABAAAABAAAAHj9//8IAAAAEAAAAAYAAABkc191aWQAAAQAAABuYW1lAAAAAAAAAABw/f//BgAAAGRzX3VpZAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAADwAAAA8AAAAAAAEATgAAAABAAAABAAAAOT9//8IAAAAEAAAAAQAAAB0YWdzAAAAAAQAAABuYW1lAAAAAAAAAADc/f//BAAAAHRhZ3MAAAAATv7//xQAAACQAAAAkAAAAAAAAAWMAAAAAgAAACgAAAAEAAAAQP7//wgAAAAMAAAAAwAAAHVybAAEAAAAbmFtZQAAAABg/v//CAAAAEAAAAA0AAAAeyJsaW5rcyI6W3sidGl0bGUiOiJsaW5rIiwidXJsIjoiJHtfX3ZhbHVlLnRleHR9In1dfQAAAAAGAAAAY29uZmlnAAAAAAAAiP7//wMAAAB1cmwA9v7//xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAADk/v//CAAAABQAAAAKAAAAcGFuZWxfdHlwZQAABAAAAG5hbWUAAAAAAAAAAOD+//8KAAAAcGFuZWxfdHlwZQAAVv///xQAAAA8AAAAPAAAAAAAAAU4AAAAAQAAAAQAAABE////CAAAABAAAAAEAAAAbmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAAPP///wQAAABuYW1lAAAAAK7///8UAAAAOAAAADgAAAAAAAAFNAAAAAEAAAAEAAAAnP///wgAAAAMAAAAAwAAAHVpZAAEAAAAbmFtZQAAAAAAAAAAkP///wMAAAB1aWQAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAGtpbmQAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAGtpbmQAAAAAAAAAAP////9YAgAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAkAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAmAEAAAEAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAACQAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAACAAAAAAAAAAAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAIAAAAAAAAADAAAAAAAAAAHwAAAAAAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAIAAAAAAAAAFgAAAAAAAAAAAAAAAAAAABYAAAAAAAAAAAAAAAAAAAAWAAAAAAAAAAIAAAAAAAAAGAAAAAAAAAABQAAAAAAAABoAAAAAAAAAAEAAAAAAAAAcAAAAAAAAAAIAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAAAEAAAAAAAAAgAAAAAAAAAAIAAAAAAAAAIgAAAAAAAAAAAAAAAAAAACIAAAAAAAAAAAAAAAAAAAAiAAAAAAAAAAIAAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAkAAABkYXNoYm9hcmQAAAAAAAAAAAAAAAEAAAAxAAAAAAAAAAAAAAAfAAAARXlqYWZqYWxsYWrDtmt1bGwgRXJ1cHRpb24gZGF0YQAAAAAAAAAAAAAAAAAFAAAAL2QvMS8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAQAAQAAAGAEAAAAAAAAYAIAAAAAAACQAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACsAAAAAwAAAFgAAAAoAAAABAAAADz8//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAXPz//wgAAAAYAAAADQAAAFF1ZXJ5IHJlc3VsdHMAAAAEAAAAbmFtZQAAAACI/P//CAAAADgAAAAuAAAAeyJ0eXBlIjoic2VhcmNoLXJlc3VsdHMiLCJjdXN0b20iOnsiY291bnQiOjF9fQAABAAAAG1ldGEAAAAACAAAAAgDAACgAgAARAIAAOABAAA0AQAA2AAAAGgAAAAEAAAAKv3//xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAAAY/f//CAAAABQAAAAIAAAAbG9jYXRpb24AAAAABAAAAG5hbWUAAAAAAAAAABT9//8IAAAAbG9jYXRpb24AAAAApv///xQAAAA8AAAAPAAAAAAABAE4AAAAAQAAAAQAAAB4/f//CAAAABAAAAAGAAAAZHNfdWlkAAAEAAAAbmFtZQAAAAAAAAAAcP3//wYAAABkc191aWQAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAAA8AAAAPAAAAAAABAE4AAAAAQAAAAQAAADk/f//CAAAABAAAAAEAAAAdGFncwAAAAAEAAAAbmFtZQAAAAAAAAAA3P3//wQAAAB0YWdzAAAAAE7+//8UAAAAkAAAAJAAAAAAAAAFjAAAAAIAAAAoAAAABAAAAED+//8IAAAADAAAAAMAAAB1cmwABAAAAG5hbWUAAAAAYP7//wgAAABAAAAANAAAAHsibGlua3MiOlt7InRpdGxlIjoibGluayIsInVybCI6IiR7X192YWx1ZS50ZXh0fSJ9XX0AAAAABgAAAGNvbmZpZwAAAAAAAIj+//8DAAAAdXJsAPb+//8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAA5P7//wgAAAAUAAAACgAAAHBhbmVsX3R5cGUAAAQAAABuYW1lAAAAAAAAAADg/v//CgAAAHBhbmVsX3R5cGUAAFb///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAARP///wgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAADz///8EAAAAbmFtZQAAAACu////FAAAADgAAAA4AAAAAAAABTQAAAABAAAABAAAAJz///8IAAAADAAAAAMAAAB1aWQABAAAAG5hbWUAAAAAAAAAAJD///8DAAAAdWlkAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABraW5kAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABraW5kAAAAAHgEAABBUlJPVzE=