From 2aae523c3f0fbf3690a30c23578170b6c515e374 Mon Sep 17 00:00:00 2001 From: Scott Lepper Date: Fri, 21 Mar 2025 09:57:20 -0400 Subject: [PATCH] Search: Fix title filter overmatching (#102547) * fix issues with over matching * search wildcard for backward compatibility --- pkg/storage/unified/search/bleve.go | 102 +++++++-- pkg/storage/unified/search/bleve_mappings.go | 16 +- .../unified/search/bleve_mappings_test.go | 7 +- .../unified/search/bleve_performance_test.go | 131 ++++++++++++ .../unified/search/bleve_search_test.go | 198 +++++++++++++++--- pkg/storage/unified/search/document_test.go | 11 +- 6 files changed, 403 insertions(+), 62 deletions(-) create mode 100644 pkg/storage/unified/search/bleve_performance_test.go diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 4976449f548..e99df0870e3 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -159,7 +159,7 @@ func (b *bleveBackend) BuildIndex(ctx context.Context, var index bleve.Index build := true - mapper, err := getBleveMappings(fields) + mapper, err := GetBleveMappings(fields) if err != nil { return nil, err } @@ -660,8 +660,14 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resource.Res } } - // Add a text query - if req.Query != "" && req.Query != "*" { + if len(req.Query) > 1 && strings.Contains(req.Query, "*") { + // wildcard query is expensive - should be used with caution + wildcard := bleve.NewWildcardQuery(req.Query) + queries = append(queries, wildcard) + } + + if req.Query != "" && !strings.Contains(req.Query, "*") { + // Add a text query searchrequest.Fields = append(searchrequest.Fields, resource.SEARCH_FIELD_SCORE) // There are multiple ways to match the query string to documents. The following queries are ordered by priority: @@ -789,6 +795,11 @@ var textSortFields = map[string]string{ const lowerCase = "phrase" +// termField fields to use termQuery for filtering +var termFields = []string{ + resource.SEARCH_FIELD_TITLE, +} + // Convert a "requirement" into a bleve query func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *resource.ErrorResult) { switch selection.Operator(req.Operator) { @@ -797,16 +808,14 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r return query.NewMatchAllQuery(), nil } - if len(req.Values[0]) == 1 { - q := query.NewMatchQuery(filterValue(req.Key, req.Values[0])) - q.FieldVal = prefix + req.Key - return q, nil + if len(req.Values) == 1 { + filter := filterValue(req.Key, req.Values[0]) + return newQuery(req.Key, filter, prefix), nil } conjuncts := []query.Query{} for _, v := range req.Values { - q := query.NewMatchQuery(filterValue(req.Key, v)) - q.FieldVal = prefix + req.Key + q := newQuery(req.Key, filterValue(req.Key, v), prefix) conjuncts = append(conjuncts, q) } @@ -822,15 +831,13 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r return query.NewMatchAllQuery(), nil } if len(req.Values) == 1 { - q := query.NewMatchQuery(filterValue(req.Key, req.Values[0])) - q.FieldVal = prefix + req.Key + q := newQuery(req.Key, filterValue(req.Key, req.Values[0]), prefix) return q, nil } disjuncts := []query.Query{} for _, v := range req.Values { - q := query.NewMatchQuery(filterValue(req.Key, v)) - q.FieldVal = prefix + req.Key + q := newQuery(req.Key, filterValue(req.Key, v), prefix) disjuncts = append(disjuncts, q) } @@ -841,7 +848,8 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r var mustNotQueries []query.Query for _, value := range req.Values { - mustNotQueries = append(mustNotQueries, bleve.NewMatchQuery(filterValue(req.Key, value))) + q := newQuery(req.Key, filterValue(req.Key, value), prefix) + mustNotQueries = append(mustNotQueries, q) } boolQuery.AddMustNot(mustNotQueries...) @@ -856,6 +864,55 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r ) } +// newQuery will create a query that will match the value or the tokens of the value +func newQuery(key string, value string, prefix string) query.Query { + if value == "*" { + return bleve.NewMatchAllQuery() + } + if strings.Contains(value, "*") { + // wildcard query is expensive - should be used with caution + return bleve.NewWildcardQuery(value) + } + delimiter, ok := hasTerms(value) + if slices.Contains(termFields, key) && ok { + return newTermsQuery(key, value, delimiter, prefix) + } + q := bleve.NewMatchQuery(value) + q.SetField(prefix + key) + return q +} + +// newTermsQuery will create a query that will match on term or tokens +func newTermsQuery(key string, value string, delimiter string, prefix string) query.Query { + tokens := strings.Split(value, delimiter) + // won't match with ending space + value = strings.TrimSuffix(value, " ") + + q := bleve.NewTermQuery(value) + q.SetField(prefix + key) + + cq := newMatchAllTokensQuery(tokens, key, prefix) + return bleve.NewDisjunctionQuery(q, cq) +} + +// newMatchAllTokensQuery will create a query that will match on all tokens +func newMatchAllTokensQuery(tokens []string, key string, prefix string) query.Query { + cq := bleve.NewConjunctionQuery() + for _, token := range tokens { + _, ok := hasTerms(token) + if ok { + tq := bleve.NewTermQuery(token) + tq.SetField(prefix + key) + cq.AddQuery(tq) + continue + } + mq := bleve.NewMatchQuery(token) + mq.SetField(prefix + key) + cq.AddQuery(mq) + } + return cq +} + // filterValue will convert the value to lower case if the field is a phrase field func filterValue(field string, v string) string { if strings.HasSuffix(field, lowerCase) { @@ -1068,3 +1125,20 @@ func (q *permissionScopedQuery) Searcher(ctx context.Context, i index.IndexReade return filteringSearcher, nil } + +// hasTerms - any value that will be split into multiple tokens +var hasTerms = func(v string) (string, bool) { + for _, c := range TermCharacters { + if strings.Contains(v, c) { + return c, true + } + } + return "", false +} + +// TermCharacters characters that will be used to determine if a value is split into tokens +var TermCharacters = []string{ + " ", "-", "_", ".", ",", ":", ";", "?", "!", "@", "#", "$", "%", "^", "&", "*", "(", ")", "+", + "=", "{", "}", "[", "]", "|", "\\", "/", "<", ">", "~", "`", + "'", "\"", +} diff --git a/pkg/storage/unified/search/bleve_mappings.go b/pkg/storage/unified/search/bleve_mappings.go index c9afce91a09..3a8c4e75420 100644 --- a/pkg/storage/unified/search/bleve_mappings.go +++ b/pkg/storage/unified/search/bleve_mappings.go @@ -9,7 +9,7 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" ) -func getBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) { +func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) { mapper := bleve.NewIndexMapping() err := RegisterCustomAnalyzers(mapper) @@ -31,22 +31,22 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM } mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_NAME, nameMapping) + // for sorting by title full phrase + titlePhraseMapping := bleve.NewKeywordFieldMapping() + titlePhraseMapping.Store = false // already stored in title + mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_PHRASE, titlePhraseMapping) + // for searching by title - uses an edge ngram token filter titleSearchMapping := bleve.NewTextFieldMapping() titleSearchMapping.Analyzer = TITLE_ANALYZER titleSearchMapping.Store = false // already stored in title - mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_NGRAM, titleSearchMapping) // mapping for title to search on words/tokens larger than the ngram size titleWordMapping := bleve.NewTextFieldMapping() titleWordMapping.Analyzer = standard.Name titleWordMapping.Store = true - mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE, titleWordMapping) - - // for filtering/sorting by title full phrase - titlePhraseMapping := bleve.NewKeywordFieldMapping() - titleSearchMapping.Store = false // already stored in title - mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_PHRASE, titlePhraseMapping) + // NOTE: this causes 3 title fields in the response + mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE, titleWordMapping, titleSearchMapping, titlePhraseMapping) descriptionMapping := &mapping.FieldMapping{ Name: resource.SEARCH_FIELD_DESCRIPTION, diff --git a/pkg/storage/unified/search/bleve_mappings_test.go b/pkg/storage/unified/search/bleve_mappings_test.go index 9c1c6c7f8ad..3b8027ee06e 100644 --- a/pkg/storage/unified/search/bleve_mappings_test.go +++ b/pkg/storage/unified/search/bleve_mappings_test.go @@ -1,4 +1,4 @@ -package search +package search_test import ( "fmt" @@ -9,10 +9,11 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/grafana/grafana/pkg/storage/unified/search" ) func TestDocumentMapping(t *testing.T) { - mappings, err := getBleveMappings(nil) + mappings, err := search.GetBleveMappings(nil) require.NoError(t, err) data := resource.IndexableDocument{ Title: "title", @@ -48,5 +49,5 @@ func TestDocumentMapping(t *testing.T) { fmt.Printf("DOC: fields %d\n", len(doc.Fields)) fmt.Printf("DOC: size %d\n", doc.Size()) - require.Equal(t, 16, len(doc.Fields)) + require.Equal(t, 17, len(doc.Fields)) } diff --git a/pkg/storage/unified/search/bleve_performance_test.go b/pkg/storage/unified/search/bleve_performance_test.go new file mode 100644 index 00000000000..2d645235259 --- /dev/null +++ b/pkg/storage/unified/search/bleve_performance_test.go @@ -0,0 +1,131 @@ +package search_test + +import ( + "context" + "fmt" + "os" + "runtime" + "testing" + "time" + + "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/stretchr/testify/require" +) + +func setupIndex() (resource.ResourceIndex, string) { + // size := 1000000 // TODO: 200k documents standard size? + size := 200000 + // batchSize := 1000 slower 8s (for 200k documents) - 34s (for 1M documents) + // batchSize := 10000 // faster 5s (for 200k documents) - 27s (for 1M documents) + batchSize := 100000 // fasterer 3.5s (for 200k documents) - 27s (for 1M documents) + writer := newTestWriter(size, batchSize) + return newTestDashboardsIndex(nil, 1, int64(size), int64(batchSize), writer) +} + +const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms) +const maxAllowedAllocMB = 1 // 1MB memory per operation +const maxAllowedAlloc = maxAllowedAllocMB * 1024 * 1024 +const verbose = false + +// BenchmarkBleveQuery measures the time, mem, cpu to execute a search query +// changes the the indexer settings can cause unforeseen performance issues ( for example: using wildcard queries ) +// this will fail if the stats exceed the "normal" thresholds +func BenchmarkBleveQuery(b *testing.B) { + var memStatsStart runtime.MemStats + var memStatsAfterIndex runtime.MemStats + runtime.ReadMemStats(&memStatsStart) + + testIndex, testIndexDir := setupIndex() + defer func() { + err := os.RemoveAll(testIndexDir) + if err != nil { + fmt.Printf("Error removing index directory: %v\n", err) + } + }() + + runtime.ReadMemStats(&memStatsAfterIndex) + + allocDiff := memStatsAfterIndex.Alloc - memStatsStart.Alloc + + logVerbose(fmt.Sprintf("Memory allocated for index: %d bytes", allocDiff)) + + searchRequest := newQueryByTitle("name99999") + + b.ResetTimer() // Reset timer before benchmarking + b.ReportAllocs() // Track memory allocations + + for i := 0; i < b.N; i++ { + start := time.Now() // Start timer + var memStatsBefore, memStatsAfter runtime.MemStats + runtime.ReadMemStats(&memStatsBefore) + + _, err := testIndex.Search(context.Background(), nil, searchRequest, nil) + + elapsed := time.Since(start) // Calculate elapsed time + runtime.ReadMemStats(&memStatsAfter) + allocDiff := (memStatsAfter.Alloc - memStatsBefore.Alloc) + if memStatsAfter.Alloc < memStatsBefore.Alloc { + // This can happen due to memory being freed after the search operation + allocDiff = 0 // don't care if it goes down + } + + logVerbose(fmt.Sprintf("Memory allocated for query: %d bytes", allocDiff)) + + require.NoError(b, err) + + // Fail if query takes longer than maxAllowedTime + if elapsed > maxAllowedTime { + b.Fatalf("Query too slow: %v (limit: %v)", elapsed, maxAllowedTime) + } + // Check memory allocation limit + if allocDiff > maxAllowedAlloc { + b.Fatalf("Excessive memory usage: %d mb (limit: %d mb)", allocDiff, maxAllowedAllocMB) + } + } +} + +func newTestWriter(size int, batchSize int) IndexWriter { + key := &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + } + + return func(index resource.ResourceIndex) (int64, error) { + total := time.Now() + start := time.Now() + for i := range size { + name := fmt.Sprintf("name%d", i) + err := index.Write(&resource.IndexableDocument{ + RV: int64(i), + Name: name, + Key: &resource.ResourceKey{ + Name: name, + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: name + "-title", + }) + if err != nil { + return 0, err + } + // show progress for every batch + if i%batchSize == 0 && verbose { + fmt.Printf("Indexed %d documents\n", i) + end := time.Now() + fmt.Printf("Time taken for indexing batch: %s\n", end.Sub(start)) + start = time.Now() + } + } + end := time.Now() + logVerbose(fmt.Sprintf("Indexed %d documents in %s", size, end.Sub(total))) + return 0, nil + } +} + +func logVerbose(msg string) { + if verbose { + fmt.Println(msg) + } +} diff --git a/pkg/storage/unified/search/bleve_search_test.go b/pkg/storage/unified/search/bleve_search_test.go index 29287de97e4..ae1782c61a2 100644 --- a/pkg/storage/unified/search/bleve_search_test.go +++ b/pkg/storage/unified/search/bleve_search_test.go @@ -1,4 +1,4 @@ -package search +package search_test import ( "context" @@ -16,8 +16,11 @@ import ( "github.com/grafana/grafana/pkg/services/store/kind/dashboard" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/grafana/grafana/pkg/storage/unified/search" ) +const threshold = 9999 + func TestCanSearchByTitle(t *testing.T) { key := &resource.ResourceKey{ Namespace: "default", @@ -26,7 +29,7 @@ func TestCanSearchByTitle(t *testing.T) { } t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -53,7 +56,7 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // search for phrase - query := newQuery("") + query := newTestQuery("") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(2), res.TotalHits) @@ -61,7 +64,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will boost phrase match query over match query results", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -88,7 +91,7 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // search for phrase - query := newQuery("want hello") + query := newTestQuery("want hello") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(2), res.TotalHits) @@ -96,7 +99,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will prioritize matches", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -122,7 +125,7 @@ func TestCanSearchByTitle(t *testing.T) { }) require.NoError(t, err) - query := newQuery("New dash") + query := newTestQuery("New dash") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(2), res.TotalHits) @@ -130,7 +133,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -157,7 +160,7 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // search for exact match - query := newQuery("we want hello") + query := newTestQuery("we want hello") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(2), res.TotalHits) @@ -165,7 +168,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title with numbers will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -180,20 +183,65 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // search for prefix of title with mix of chars and numbers - query := newQuery("A12") + query := newQueryByTitle("A12") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // search for whole title - query = newQuery("A123456") + query = newQueryByTitle("A123456") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + // case insensive search for partial title + query = newQueryByTitle("a1234") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) }) + t.Run("title will match escaped characters", func(t *testing.T) { + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "aaa", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "what\"s up", + }) + require.NoError(t, err) + + err = index.Write(&resource.IndexableDocument{ + RV: 2, + Name: "name2", + Key: &resource.ResourceKey{ + Name: "name2", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "what\"s that", + }) + require.NoError(t, err) + + query := newQueryByTitle("what\"s up") + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(1), res.TotalHits) + + query = newQueryByTitle("what\"s") + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(2), res.TotalHits) + }) + t.Run("title search will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -208,50 +256,50 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // search by entire phrase - query := newQuery("I want to say a wonderfully Hello to the WORLD! Hello-world") + query := newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // search for word at start - query = newQuery("hello") + query = newTestQuery("hello") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // search for word larger than ngram max size - query = newQuery("wonderfully") + query = newQueryByTitle("wonderfully") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // search for word at end - query = newQuery("world") + query = newQueryByTitle("world") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // can search for word substring anchored at start of word (edge ngram) - query = newQuery("worl") + query = newQueryByTitle("worl") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // can search for multiple, non-consecutive words in title - query = newQuery("hello world") + query = newQueryByTitle("hello world") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) // can search for a term with a hyphen - query = newQuery("hello-world") + query = newQueryByTitle("hello-world") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(1), res.TotalHits) }) t.Run("title search will NOT match documents", func(t *testing.T) { - index := newTestDashboardsIndex(t) + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) err := index.Write(&resource.IndexableDocument{ RV: 1, Name: "name1", @@ -290,26 +338,79 @@ func TestCanSearchByTitle(t *testing.T) { require.NoError(t, err) // word that doesn't exist - query := newQuery("cats") + query := newQueryByTitle("cats") res, err := index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(0), res.TotalHits) // string shorter than 3 chars (ngam min) - query = newQuery("ma") + query = newQueryByTitle("ma") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(0), res.TotalHits) // substring that doesn't exist - query = newQuery("A01") + query = newQueryByTitle("A01") res, err = index.Search(context.Background(), nil, query, nil) require.NoError(t, err) require.Equal(t, int64(0), res.TotalHits) }) + + t.Run("title search with character will match one document", func(t *testing.T) { + index, _ := newTestDashboardsIndex(t, threshold, 2, 2, noop) + err := index.Write(&resource.IndexableDocument{ + RV: 1, + Name: "name1", + Key: &resource.ResourceKey{ + Name: "aaa", + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: "foo", + }) + require.NoError(t, err) + + for i, v := range search.TermCharacters { + err = index.Write(&resource.IndexableDocument{ + RV: int64(i), + Name: fmt.Sprintf("name%d", i), + Key: &resource.ResourceKey{ + Name: fmt.Sprintf("name%d", i), + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: fmt.Sprintf(`test foo%d%sbar`, i, v), + }) + require.NoError(t, err) + } + + for i, v := range search.TermCharacters { + title := fmt.Sprintf(`test foo%d%sbar`, i, v) + query := newQueryByTitle(title) + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + if res.TotalHits != 1 { + fmt.Printf("i: %d, v: %s, title: %s", i, v, title) + } + require.Equal(t, int64(1), res.TotalHits) + + // can search for a title with a term character suffix + title = fmt.Sprintf(`foo%d%s`, i, v) + query = newQueryByTitle(title) + res, err = index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + if res.TotalHits != 1 { + fmt.Printf("i: %d, v: %s, title: %s", i, v, title) + } + + require.Equal(t, int64(1), res.TotalHits) + } + }) } -func newQuery(query string) *resource.ResourceSearchRequest { +func newTestQuery(query string) *resource.ResourceSearchRequest { return &resource.ResourceSearchRequest{ Options: &resource.ListOptions{ Key: &resource.ResourceKey{ @@ -323,7 +424,21 @@ func newQuery(query string) *resource.ResourceSearchRequest { } } -func newTestDashboardsIndex(t *testing.T) resource.ResourceIndex { +func newQueryByTitle(query string) *resource.ResourceSearchRequest { + return &resource.ResourceSearchRequest{ + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Namespace: "default", + Group: "dashboard.grafana.app", + Resource: "dashboards", + }, + Fields: []*resource.Requirement{{Key: "title", Operator: "=", Values: []string{query}}}, + }, + Limit: 100000, + } +} + +func newTestDashboardsIndex(t TB, threshold int64, size int64, batchSize int64, writer IndexWriter) (resource.ResourceIndex, string) { key := &resource.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", @@ -332,17 +447,18 @@ func newTestDashboardsIndex(t *testing.T) resource.ResourceIndex { tmpdir, err := os.MkdirTemp("", "grafana-bleve-test") require.NoError(t, err) - backend, err := NewBleveBackend(BleveOptions{ + backend, err := search.NewBleveBackend(search.BleveOptions{ Root: tmpdir, - FileThreshold: 9999, // use in-memory for tests + FileThreshold: threshold, // use in-memory for tests + BatchSize: int(batchSize), }, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorageSearchPermissionFiltering), nil) require.NoError(t, err) rv := int64(10) ctx := identity.WithRequester(context.Background(), &user.SignedInUser{Namespace: "ns"}) - info, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { - return &DashboardDocumentBuilder{ + info, err := search.DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &search.DashboardDocumentBuilder{ Namespace: namespace, Blob: blob, Stats: make(map[string]map[string]int64), // empty stats @@ -355,10 +471,16 @@ func newTestDashboardsIndex(t *testing.T) resource.ResourceIndex { Namespace: key.Namespace, Group: key.Group, Resource: key.Resource, - }, 2, rv, info.Fields, func(index resource.ResourceIndex) (int64, error) { return 0, nil }) + }, size, rv, info.Fields, writer) require.NoError(t, err) - return index + return index, tmpdir +} + +type IndexWriter func(index resource.ResourceIndex) (int64, error) + +var noop IndexWriter = func(index resource.ResourceIndex) (int64, error) { + return 0, nil } // helper to check which tokens are generated by an analyzer @@ -399,3 +521,15 @@ func debugIndexedTerms(index bleve.Index, field string) { } } } + +// TB is an interface that works for both *testing.T and *testing.B +type TB interface { + Log(args ...interface{}) + Logf(format string, args ...interface{}) + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Helper() + FailNow() +} diff --git a/pkg/storage/unified/search/document_test.go b/pkg/storage/unified/search/document_test.go index c11ada997a6..49c7e0c9e89 100644 --- a/pkg/storage/unified/search/document_test.go +++ b/pkg/storage/unified/search/document_test.go @@ -1,4 +1,4 @@ -package search +package search_test import ( "context" @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/store/kind/dashboard" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/grafana/grafana/pkg/storage/unified/search" ) func doSnapshotTests(t *testing.T, builder resource.DocumentBuilder, kind string, key *resource.ResourceKey, names []string) { @@ -52,14 +53,14 @@ func TestDashboardDocumentBuilder(t *testing.T) { Resource: "dashboards", } - info, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { - return &DashboardDocumentBuilder{ + info, err := search.DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { + return &search.DashboardDocumentBuilder{ Namespace: namespace, Blob: blob, Stats: map[string]map[string]int64{ "aaa": { - DASHBOARD_ERRORS_LAST_1_DAYS: 1, - DASHBOARD_ERRORS_LAST_7_DAYS: 1, + search.DASHBOARD_ERRORS_LAST_1_DAYS: 1, + search.DASHBOARD_ERRORS_LAST_7_DAYS: 1, }, }, DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{