From 827da46c5116b02ad85cf2aadbf6a2f0ab4ab7aa Mon Sep 17 00:00:00 2001 From: owensmallwood Date: Fri, 7 Mar 2025 09:51:15 -0600 Subject: [PATCH] Unified Storage: Remove wildcard queries (#101441) * WIP adding custom analyzer so we can do substring search efficiently * Adding unit tests for title search * formatting * adds more title search unit tests * organize helpers * fixes issue caused by having two title mappings * Removes camelcase token filter since it prevents you from searching for a substring of chars and numbers. Adds regression test. * adds back mapping for title_phrase * use simple analyzer for input query string so it doesn't filter out english stop words * ran bleve tests, table snapshots updated * ignore linter for "unused" test functions. They are very helpful for troubleshooting search. Keeping them. * only log total hits and query cost if result not nil * fixes failing test - one more field because there are two title mappings now * fix test * fixes test - only take first item when its the title * Adds separate internal field for title ngram mapping. When searching with a query, results are sorted by score desc. When searching without a query, results are sorted by title desc. Adjusts ngram max to be 10. Text queries are a disjunction of an exact match, phrase match, and a match. Boosted to have priority in that order. Adds more unit tests for searching. * linter * fix test * ran tests - generated new test dash json * sort by title phrase instead of title * fix test - not relying on /apis/dashboard/search to apply title sorting anymore --- pkg/registry/apis/dashboard/search.go | 8 +- pkg/registry/apis/dashboard/search_test.go | 14 +- pkg/storage/unified/resource/document.go | 7 +- pkg/storage/unified/resource/document_test.go | 1 + pkg/storage/unified/search/bleve.go | 48 +- pkg/storage/unified/search/bleve_mappings.go | 28 +- .../unified/search/bleve_mappings_test.go | 7 +- .../unified/search/bleve_search_test.go | 426 ++++++++++++++++++ .../unified/search/custom_analyzers.go | 48 ++ .../testdata/doc/dashboard-aaa-out.json | 1 + .../search/testdata/doc/folder-aaa-out.json | 1 + .../search/testdata/doc/folder-bbb-out.json | 1 + .../search/testdata/doc/playlist-aaa-out.json | 1 + .../search/testdata/doc/report-aaa-out.json | 1 + 14 files changed, 561 insertions(+), 31 deletions(-) create mode 100644 pkg/storage/unified/search/bleve_search_test.go create mode 100644 pkg/storage/unified/search/custom_analyzers.go diff --git a/pkg/registry/apis/dashboard/search.go b/pkg/registry/apis/dashboard/search.go index 7ade0969ead..ffa9de3b04c 100644 --- a/pkg/registry/apis/dashboard/search.go +++ b/pkg/registry/apis/dashboard/search.go @@ -369,6 +369,10 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) { return } + if result != nil { + s.log.Debug("search result hits and cost", "total_hits", result.TotalHits, "query_cost", result.QueryCost) + } + parsedResults, err := dashboardsearch.ParseResults(result, searchRequest.Offset) if err != nil { errhttp.Write(ctx, err, w) @@ -378,8 +382,8 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) { if len(searchRequest.SortBy) == 0 { // default sort by resource descending ( folders then dashboards ) then title sort.Slice(parsedResults.Hits, func(i, j int) bool { - return parsedResults.Hits[i].Resource > parsedResults.Hits[j].Resource || - (parsedResults.Hits[i].Resource == parsedResults.Hits[j].Resource && strings.ToLower(parsedResults.Hits[i].Title) < strings.ToLower(parsedResults.Hits[j].Title)) + // Just sorting by resource for now. The rest should be sorted by search score already + return parsedResults.Hits[i].Resource > parsedResults.Hits[j].Resource }) } diff --git a/pkg/registry/apis/dashboard/search_test.go b/pkg/registry/apis/dashboard/search_test.go index 16460c87a69..70a669b5c55 100644 --- a/pkg/registry/apis/dashboard/search_test.go +++ b/pkg/registry/apis/dashboard/search_test.go @@ -276,7 +276,7 @@ func TestSearchHandler(t *testing.T) { } }) - t.Run("Sort - default sort by resource then title", func(t *testing.T) { + t.Run("Sort - default sort by resource", func(t *testing.T) { rows := make([]*resource.ResourceTableRow, len(mockResults)) for i, r := range mockResults { rows[i] = &resource.ResourceTableRow{ @@ -334,7 +334,7 @@ func TestSearchHandler(t *testing.T) { err := json.NewDecoder(resp.Body).Decode(p) require.NoError(t, err) assert.Equal(t, len(mockResults), len(p.Hits)) - assert.Equal(t, mockResults[3].Value, p.Hits[0].Title) + assert.Equal(t, mockResults[2].Value, p.Hits[0].Title) assert.Equal(t, mockResults[1].Value, p.Hits[3].Title) }) } @@ -646,16 +646,16 @@ var mockResults = []MockResult{ Resource: "dashboard", Value: "Dashboard 2", }, - { - Name: "f2", - Resource: "folder", - Value: "Folder 2", - }, { Name: "f1", Resource: "folder", Value: "Folder 1", }, + { + Name: "f2", + Resource: "folder", + Value: "Folder 2", + }, } func (m *MockClient) Search(ctx context.Context, in *resource.ResourceSearchRequest, opts ...grpc.CallOption) (*resource.ResourceSearchResponse, error) { diff --git a/pkg/storage/unified/resource/document.go b/pkg/storage/unified/resource/document.go index 2229527bb56..c8d3b72072b 100644 --- a/pkg/storage/unified/resource/document.go +++ b/pkg/storage/unified/resource/document.go @@ -62,6 +62,9 @@ type IndexableDocument struct { // The generic display name Title string `json:"title,omitempty"` + // internal field for searching title with ngrams + TitleNgram string `json:"title_ngram,omitempty"` + // internal sort field for title ( don't set this directly ) TitlePhrase string `json:"title_phrase,omitempty"` @@ -169,7 +172,8 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces Key: key, RV: rv, Name: key.Name, - Title: title, // We always want *something* to display + Title: title, // We always want *something* to display + TitleNgram: title, TitlePhrase: strings.ToLower(title), // Lowercase for case-insensitive sorting Labels: obj.GetLabels(), Folder: obj.GetFolder(), @@ -264,6 +268,7 @@ const SEARCH_FIELD_NAMESPACE = "namespace" const SEARCH_FIELD_NAME = "name" const SEARCH_FIELD_RV = "rv" const SEARCH_FIELD_TITLE = "title" +const SEARCH_FIELD_TITLE_NGRAM = "title_ngram" const SEARCH_FIELD_TITLE_PHRASE = "title_phrase" // filtering/sorting on title by full phrase const SEARCH_FIELD_DESCRIPTION = "description" const SEARCH_FIELD_TAGS = "tags" diff --git a/pkg/storage/unified/resource/document_test.go b/pkg/storage/unified/resource/document_test.go index 2135748533d..8c14550578b 100644 --- a/pkg/storage/unified/resource/document_test.go +++ b/pkg/storage/unified/resource/document_test.go @@ -37,6 +37,7 @@ func TestStandardDocumentBuilder(t *testing.T) { "rv": 10, "title": "test playlist unified storage", "title_phrase": "test playlist unified storage", + "title_ngram": "test playlist unified storage", "created": 1717236672000, "createdBy": "user:ABC", "updatedBy": "user:XYZ", diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 73eee078f69..621e6c3c1b5 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -13,6 +13,8 @@ import ( "time" "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/query" @@ -118,7 +120,10 @@ func (b *bleveBackend) BuildIndex(ctx context.Context, var index bleve.Index build := true - mapper := getBleveMappings(fields) + mapper, err := getBleveMappings(fields) + if err != nil { + return nil, err + } if size > b.opts.FileThreshold { resourceDir := filepath.Join(b.opts.Root, key.Namespace, @@ -596,12 +601,26 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resource.Res // Add a text query if req.Query != "" && req.Query != "*" { searchrequest.Fields = append(searchrequest.Fields, resource.SEARCH_FIELD_SCORE) - // mimic the behavior of the sql search - query := strings.ToLower(req.Query) - if !strings.Contains(query, "*") { - query = "*" + query + "*" - } - queries = append(queries, bleve.NewWildcardQuery(query)) + + // There are multiple ways to match the query string to documents. The following queries are ordered by priority: + + // Query 1: Match the exact query string + queryExact := bleve.NewMatchQuery(req.Query) + queryExact.SetBoost(10.0) + queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token + + // Query 2: Phrase query with standard analyzer + queryPhrase := bleve.NewMatchPhraseQuery(req.Query) + queryExact.SetBoost(5.0) + queryPhrase.Analyzer = standard.Name + + // Query 3: Match query with standard analyzer + queryAnalyzed := bleve.NewMatchQuery(req.Query) + queryAnalyzed.Analyzer = standard.Name + + // At least one of the queries must match + searchQuery := bleve.NewDisjunctionQuery(queryExact, queryAnalyzed, queryPhrase) + queries = append(queries, searchQuery) } switch len(queries) { @@ -664,11 +683,18 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resource.Res sorting := getSortFields(req) searchrequest.SortBy(sorting) - // Always sort by *something*, otherwise the order is unstable + // When no sort fields are provided, sort by score if there is a query, otherwise sort by title if len(sorting) == 0 { - searchrequest.Sort = append(searchrequest.Sort, &search.SortDocID{ - Desc: false, - }) + if req.Query != "" && req.Query != "*" { + searchrequest.Sort = append(searchrequest.Sort, &search.SortScore{ + Desc: true, + }) + } else { + searchrequest.Sort = append(searchrequest.Sort, &search.SortField{ + Field: resource.SEARCH_FIELD_TITLE_PHRASE, + Desc: false, + }) + } } return searchrequest, nil diff --git a/pkg/storage/unified/search/bleve_mappings.go b/pkg/storage/unified/search/bleve_mappings.go index 7c59bbae559..8ffceeca568 100644 --- a/pkg/storage/unified/search/bleve_mappings.go +++ b/pkg/storage/unified/search/bleve_mappings.go @@ -3,15 +3,22 @@ package search import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/mapping" "github.com/grafana/grafana/pkg/storage/unified/resource" ) -func getBleveMappings(fields resource.SearchableDocumentFields) mapping.IndexMapping { +func getBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) { mapper := bleve.NewIndexMapping() + + err := RegisterCustomAnalyzers(mapper) + if err != nil { + return nil, err + } mapper.DefaultMapping = getBleveDocMappings(fields) - return mapper + + return mapper, nil } func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentMapping { @@ -24,15 +31,22 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM } mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_NAME, nameMapping) + // for searching by title - uses an edge ngram token filter + titleSearchMapping := bleve.NewTextFieldMapping() + titleSearchMapping.Analyzer = TITLE_ANALYZER + titleSearchMapping.Store = true + mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_NGRAM, titleSearchMapping) + + // mapping for title to search on words/tokens larger than the ngram size + titleWordMapping := bleve.NewTextFieldMapping() + titleWordMapping.Analyzer = standard.Name + titleWordMapping.Store = true + mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE, titleWordMapping) + // for filtering/sorting by title full phrase titlePhraseMapping := bleve.NewKeywordFieldMapping() mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_PHRASE, titlePhraseMapping) - // for searching by title - // TODO: do we still need this since we have SEARCH_FIELD_TITLE_PHRASE? - titleSearchMapping := bleve.NewTextFieldMapping() - mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE, titleSearchMapping) - descriptionMapping := &mapping.FieldMapping{ Name: resource.SEARCH_FIELD_DESCRIPTION, Type: "text", diff --git a/pkg/storage/unified/search/bleve_mappings_test.go b/pkg/storage/unified/search/bleve_mappings_test.go index 7a563fff827..e4aa1165a8a 100644 --- a/pkg/storage/unified/search/bleve_mappings_test.go +++ b/pkg/storage/unified/search/bleve_mappings_test.go @@ -12,7 +12,8 @@ import ( ) func TestDocumentMapping(t *testing.T) { - mappings := getBleveMappings(nil) + mappings, err := getBleveMappings(nil) + require.NoError(t, err) data := resource.IndexableDocument{ Title: "title", Description: "descr", @@ -37,7 +38,7 @@ func TestDocumentMapping(t *testing.T) { } doc := document.NewDocument("id") - err := mappings.MapDocument(doc, data) + err = mappings.MapDocument(doc, data) require.NoError(t, err) for _, f := range doc.Fields { @@ -46,5 +47,5 @@ func TestDocumentMapping(t *testing.T) { fmt.Printf("DOC: fields %d\n", len(doc.Fields)) fmt.Printf("DOC: size %d\n", doc.Size()) - require.Equal(t, 14, len(doc.Fields)) + require.Equal(t, 15, len(doc.Fields)) } diff --git a/pkg/storage/unified/search/bleve_search_test.go b/pkg/storage/unified/search/bleve_search_test.go new file mode 100644 index 00000000000..ce4e8814916 --- /dev/null +++ b/pkg/storage/unified/search/bleve_search_test.go @@ -0,0 +1,426 @@ +package search + +import ( + "context" + "fmt" + "log" + "os" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/store/kind/dashboard" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/stretchr/testify/require" +) + +func TestCanSearchByTitle(t *testing.T) { + key := &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + } + + t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "name1", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "bbb", + TitleNgram: "bbb", + TitlePhrase: "bbb", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "aaa", + TitleNgram: "aaa", + TitlePhrase: "aaa", + }) + require.NoError(t, err) + + // search for phrase + query := newQuery("") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(2), res.TotalHits) + require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + }) + + t.Run("will boost phrase match query over match query results", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "name1", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "I want to say a hello", + TitleNgram: "I want to say a hello", + TitlePhrase: "I want to say a hello", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "we want hello", + TitleNgram: "we want hello", + TitlePhrase: "we want hello", + }) + require.NoError(t, err) + + // search for phrase + query := newQuery("want hello") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(2), res.TotalHits) + require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + }) + + t.Run("will prioritize matches", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "name1", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "Asserts Dashboards", + TitleNgram: "Asserts Dashboards", + TitlePhrase: "Asserts Dashboards", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "New dashboard 10", + TitleNgram: "New dashboard 10", + TitlePhrase: "New dashboard 10", + }) + require.NoError(t, err) + + query := newQuery("New dash") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(2), res.TotalHits) + require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + }) + + t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "name1", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "we want hello pls", + TitleNgram: "we want hello pls", + TitlePhrase: "we want hello pls", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "we want hello", + TitleNgram: "we want hello", + TitlePhrase: "we want hello", + }) + require.NoError(t, err) + + // search for exact match + query := newQuery("we want hello") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(2), res.TotalHits) + require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + }) + + t.Run("title with numbers will match document", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "aaa", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "A123456", + TitleNgram: "A123456", + }) + require.NoError(t, err) + + // search for prefix of title with mix of chars and numbers + query := newQuery("A12") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // search for whole title + query = newQuery("A123456") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + }) + + t.Run("title search will match document", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "aaa", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "I want to say a wonderfully Hello to the WORLD! Hello-world", + TitleNgram: "I want to say a wonderfully Hello to the WORLD! Hello-world", + TitlePhrase: "I want to say a wonderfully Hello to the WORLD! Hello-world", + }) + require.NoError(t, err) + + // search by entire phrase + query := newQuery("I want to say a wonderfully Hello to the WORLD! Hello-world") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // search for word at start + query = newQuery("hello") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // search for word larger than ngram max size + query = newQuery("wonderfully") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // search for word at end + query = newQuery("world") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // can search for word substring anchored at start of word (edge ngram) + query = newQuery("worl") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // can search for multiple, non-consecutive words in title + query = newQuery("hello world") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // can search for a term with a hyphen + query = newQuery("hello-world") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + }) + + t.Run("title search will NOT match documents", func(t *testing.T) { + index := newTestDashboardsIndex(t) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "name1", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "I want to say a wonderful Hello to the WORLD! Hello-world", + TitleNgram: "I want to say a wonderful Hello to the WORLD! Hello-world", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "A0456", + TitleNgram: "A0456", + }) + require.NoError(t, err) + err = index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name3", + Key: &resource.ResourceKey{ + Name: "name3", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "mash-A02382-10", + TitleNgram: "mash-A02382-10", + TitlePhrase: "mash-A02382-10", + }) + require.NoError(t, err) + + // word that doesn't exist + query := newQuery("cats") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(0), res.TotalHits) + + // string shorter than 3 chars (ngam min) + query = newQuery("ma") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(0), res.TotalHits) + + // substring that doesn't exist + query = newQuery("A01") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(0), res.TotalHits) + }) +} + +func newQuery(query string) *resource.ResourceSearchRequest { + return &resource.ResourceSearchRequest{ + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + }, + }, + Limit: 100000, + Query: query, + } +} + +func newTestDashboardsIndex(t *testing.T) resource.ResourceIndex { + key := &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + } + tmpdir, err := os.MkdirTemp("", "grafana-bleve-test") + require.NoError(t, err) + + backend, err := NewBleveBackend(BleveOptions{ + Root: tmpdir, + FileThreshold: 9999, // use in-memory for tests + }, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorageSearchPermissionFiltering)) + require.NoError(t, err) + + // AVOID NPE in test + resource.NewIndexMetrics(backend.opts.Root, backend) + + rv := int64(10) + ctx := identity.WithRequester(context.Background(), &user.SignedInUser{Namespace: "ns"}) + + info, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &DashboardDocumentBuilder{ + Namespace: namespace, + Blob: blob, + Stats: make(map[string]map[string]int64), // empty stats + DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{}}), + }, nil + }) + require.NoError(t, err) + + index, err := backend.BuildIndex(ctx, resource.NamespacedResource{ + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, 2, rv, info.Fields, func(index resource.ResourceIndex) (int64, error) { return 0, nil }) + require.NoError(t, err) + + return index +} + +// helper to check which tokens are generated by an analyzer +// nolint:unused +func debugAnalyzer(index bleve.Index, analyzerName string, text string) { + // Get the analyzer (default: "standard") + analyzer := index.Mapping().AnalyzerNamed(analyzerName) + if analyzer == nil { + log.Fatal("Analyzer not found") + } + + // Analyze text to see generated tokens + analysisResult := analyzer.Analyze([]byte(text)) + + // Print tokens + fmt.Println("Generated tokens for analyzer:", analyzerName) + for _, token := range analysisResult { + fmt.Println(string(token.Term)) + } +} + +// helper to check which terms are indexed for a field +// nolint:unused +func debugIndexedTerms(index bleve.Index, field string) { + // Check what terms exist for the title field + fieldTerms, err := index.FieldDict(field) + if err != nil { + log.Fatal(err) + } + + for { + term, err := fieldTerms.Next() + if err != nil { + break + } + if term != nil { + fmt.Println(term.Term) + } + } +} diff --git a/pkg/storage/unified/search/custom_analyzers.go b/pkg/storage/unified/search/custom_analyzers.go new file mode 100644 index 00000000000..51e2176298b --- /dev/null +++ b/pkg/storage/unified/search/custom_analyzers.go @@ -0,0 +1,48 @@ +package search + +import ( + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/token/edgengram" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/unique" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace" + "github.com/blevesearch/bleve/v2/mapping" +) + +const TITLE_ANALYZER = "title_analyzer" + +func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl) error { + return registerTitleAnalyzer(mapper) +} + +// The registerTitleAnalyzer function defines a custom analyzer for the title field. +// The edgeNgramTokenFilter will create n-grams anchored to the front of each token. +// For example, the token "hello" will be tokenized into "hel", "hell", "hello". +func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error { + // Define an N-Gram tokenizer (for substring search) + edgeNgramTokenFilter := map[string]interface{}{ + "type": edgengram.Name, + "min": 3.0, + "max": 10.0, + "back": edgengram.FRONT, + } + err := mapper.AddCustomTokenFilter("edge_ngram_filter", edgeNgramTokenFilter) + if err != nil { + return err + } + + //Create a custom analyzer using the N-Gram tokenizer + ngramAnalyzer := map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []string{"edge_ngram_filter", lowercase.Name, unique.Name}, + //"char_filters": //TODO IF NEEDED + } + + err = mapper.AddCustomAnalyzer(TITLE_ANALYZER, ngramAnalyzer) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json b/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json index 9fa0a53b2c1..1edd60a2ec9 100644 --- a/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json +++ b/pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json @@ -8,6 +8,7 @@ "name": "aaa", "rv": 1234, "title": "Test title", + "title_ngram": "Test title", "title_phrase": "test title", "description": "test description", "tags": [ diff --git a/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json b/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json index c2807e7f48a..aff5002f8b6 100644 --- a/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json +++ b/pkg/storage/unified/search/testdata/doc/folder-aaa-out.json @@ -8,6 +8,7 @@ "name": "aaa", "rv": 1234, "title": "test-aaa", + "title_ngram": "test-aaa", "title_phrase": "test-aaa", "created": 1730490142000, "createdBy": "user:1", diff --git a/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json b/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json index cabacb8b404..5ff5292aa96 100644 --- a/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json +++ b/pkg/storage/unified/search/testdata/doc/folder-bbb-out.json @@ -8,6 +8,7 @@ "name": "bbb", "rv": 1234, "title": "test-bbb", + "title_ngram": "test-bbb", "title_phrase": "test-bbb", "created": 1730490142000, "createdBy": "user:1", diff --git a/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json b/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json index 0bc5b824883..cb49b90a95a 100644 --- a/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json +++ b/pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json @@ -8,6 +8,7 @@ "name": "aaa", "rv": 1234, "title": "Test AAA", + "title_ngram": "Test AAA", "title_phrase": "test aaa", "created": 1731336353000, "createdBy": "user:t000000001" diff --git a/pkg/storage/unified/search/testdata/doc/report-aaa-out.json b/pkg/storage/unified/search/testdata/doc/report-aaa-out.json index 656f57c46f1..0eab64c17e5 100644 --- a/pkg/storage/unified/search/testdata/doc/report-aaa-out.json +++ b/pkg/storage/unified/search/testdata/doc/report-aaa-out.json @@ -8,6 +8,7 @@ "name": "aaa", "rv": 1234, "title": "Test AAA", + "title_ngram": "Test AAA", "title_phrase": "test aaa", "labels": { "grafana.app/deprecatedInternalID": "123"