From bd529226a355d729b8f1237d421a018a42cdc85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Fri, 5 Sep 2025 13:00:54 +0200 Subject: [PATCH] unified-storage search: return results matching all search terms (#110672) * Extract common indexing code. * Extract common search code. * Match all terms in the search query. --- pkg/storage/unified/search/bleve.go | 6 +- .../unified/search/bleve_search_test.go | 485 ++++-------------- 2 files changed, 98 insertions(+), 393 deletions(-) diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 2de6e3a1fe2..44fdffe27a8 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -1072,16 +1072,18 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R // Query 1: Match the exact query string queryExact := bleve.NewMatchQuery(req.Query) queryExact.SetBoost(10.0) - queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token + queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token + queryExact.Operator = query.MatchQueryOperatorAnd // This doesn't make a difference for keyword analyzer, we add it just to be explicit. // Query 2: Phrase query with standard analyzer queryPhrase := bleve.NewMatchPhraseQuery(req.Query) - queryExact.SetBoost(5.0) + queryPhrase.SetBoost(5.0) queryPhrase.Analyzer = standard.Name // Query 3: Match query with standard analyzer queryAnalyzed := bleve.NewMatchQuery(req.Query) queryAnalyzed.Analyzer = standard.Name + queryAnalyzed.Operator = query.MatchQueryOperatorAnd // Make sure all terms from the query are matched // At least one of the queries must match searchQuery := bleve.NewDisjunctionQuery(queryExact, queryAnalyzed, queryPhrase) diff --git a/pkg/storage/unified/search/bleve_search_test.go b/pkg/storage/unified/search/bleve_search_test.go index b8d2e572f82..423355b28c5 100644 --- a/pkg/storage/unified/search/bleve_search_test.go +++ b/pkg/storage/unified/search/bleve_search_test.go @@ -21,8 +21,39 @@ import ( const threshold = 9999 +func indexDocumentsWithTitles(t *testing.T, index resource.ResourceIndex, key resource.NamespacedResource, docsWithTitles map[string]string) { + items := make([]*resource.BulkIndexItem, 0, len(docsWithTitles)) + for name, title := range docsWithTitles { + items = append(items, &resource.BulkIndexItem{ + Action: resource.ActionIndex, + Doc: &resource.IndexableDocument{ + RV: 1, + Name: name, + Key: &resourcepb.ResourceKey{ + Name: name, + Namespace: key.Namespace, + Group: key.Group, + Resource: key.Resource, + }, + Title: title, + }, + }) + } + req := &resource.BulkIndexRequest{Items: items} + require.NoError(t, index.BulkIndex(req)) +} + +func checkSearchQuery(t *testing.T, index resource.ResourceIndex, query *resourcepb.ResourceSearchRequest, orderedExpectedNames []string) { + res, err := index.Search(context.Background(), nil, query, nil) + require.NoError(t, err) + require.Equal(t, int64(len(orderedExpectedNames)), res.TotalHits) + for ix, name := range orderedExpectedNames { + require.Equal(t, name, res.Results.Rows[ix].Key.Name) + } +} + func TestCanSearchByTitle(t *testing.T) { - key := &resourcepb.ResourceKey{ + key := resource.NamespacedResource{ Namespace: "default", Group: "dashboard.grafana.app", Resource: "dashboards", @@ -30,463 +61,135 @@ func TestCanSearchByTitle(t *testing.T) { t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "name1", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "bbb", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "aaa", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "bbb", + "name2": "aaa", }) - require.NoError(t, err) - // search for phrase - query := newTestQuery("") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(2), res.TotalHits) - require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"}) }) t.Run("will boost phrase match query over match query results", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "name1", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "I want to say a hello", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "we want hello", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a hello", + "name2": "we want hello", }) - require.NoError(t, err) - // search for phrase - query := newTestQuery("want hello") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(2), res.TotalHits) - require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"}) }) t.Run("will prioritize matches", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "name1", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "Asserts Dashboards", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "New dashboard 10", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "Asserts Dashboards", + "name2": "New dashboard 10", }) - require.NoError(t, err) - query := newTestQuery("New dash") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(2), res.TotalHits) - require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"}) + }) + + t.Run("all terms must match", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "Dashboard", + "name2": "New dashboard 10", + }) + + checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"}) }) t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "name1", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "we want hello pls", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "we want hello", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "we want hello pls", + "name2": "we want hello", }) - require.NoError(t, err) - // search for exact match - query := newTestQuery("we want hello") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(2), res.TotalHits) - require.Equal(t, "name2", res.Results.Rows[0].Key.Name) + checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"}) }) t.Run("title with numbers will match document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "aaa", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "A123456", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "A123456", }) - require.NoError(t, err) // search for prefix of title with mix of chars and numbers - query := newQueryByTitle("A12") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"}) // search for whole title - query = newQueryByTitle("A123456") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"}) // case insensive search for partial title - query = newQueryByTitle("a1234") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) + checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"}) }) t.Run("title will match escaped characters", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "aaa", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "what\"s up", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 2, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "what\"s that", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "what\"s up", + "name2": "what\"s that", }) - require.NoError(t, err) - query := newQueryByTitle("what\"s up") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - - query = newQueryByTitle("what\"s") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(2), res.TotalHits) + checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"}) + checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"}) }) t.Run("title search will match document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "aaa", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "I want to say a wonderfully Hello to the WORLD! Hello-world", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", }) - require.NoError(t, err) // search by entire phrase - query := newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) + checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"}) // search for word at start - query = newTestQuery("hello") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"}) // search for word larger than ngram max size - query = newQueryByTitle("wonderfully") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"}) // search for word at end - query = newQueryByTitle("world") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"}) // can search for word substring anchored at start of word (edge ngram) - query = newQueryByTitle("worl") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"}) // can search for multiple, non-consecutive words in title - query = newQueryByTitle("hello world") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) - - // can search for a term with a hyphen - query = newQueryByTitle("hello-world") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(1), res.TotalHits) + checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"}) + // can search for multiple, non-consecutive words in title + checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"}) }) t.Run("title search will NOT match documents", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "name1", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "I want to say a wonderful Hello to the WORLD! Hello-world", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name2", - Key: &resourcepb.ResourceKey{ - Name: "name2", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "A0456", - }, - }, - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name3", - Key: &resourcepb.ResourceKey{ - Name: "name3", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "mash-A02382-10", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", + "name2": "A0456", + "name3": "mash-A02382-10", }) - require.NoError(t, err) // word that doesn't exist - query := newQueryByTitle("cats") - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(0), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("cats"), nil) // string shorter than 3 chars (ngam min) - query = newQueryByTitle("ma") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(0), res.TotalHits) - + checkSearchQuery(t, index, newTestQuery("ma"), nil) // substring that doesn't exist - query = newQueryByTitle("A01") - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - require.Equal(t, int64(0), res.TotalHits) + checkSearchQuery(t, index, newTestQuery("A01"), nil) }) t.Run("title search with character will match one document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, 2, noop) - err := index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: 1, - Name: "name1", - Key: &resourcepb.ResourceKey{ - Name: "aaa", - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: "foo", - }, - }, - }, + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "foo", }) - require.NoError(t, err) for i, v := range search.TermCharacters { - err = index.BulkIndex(&resource.BulkIndexRequest{ - Items: []*resource.BulkIndexItem{ - { - Action: resource.ActionIndex, - Doc: &resource.IndexableDocument{ - RV: int64(i), - Name: fmt.Sprintf("name%d", i), - Key: &resourcepb.ResourceKey{ - Name: fmt.Sprintf("name%d", i), - Namespace: key.Namespace, - Group: key.Group, - Resource: key.Resource, - }, - Title: fmt.Sprintf(`test foo%d%sbar`, i, v), - }, - }, - }, - }) - require.NoError(t, err) - + name := fmt.Sprintf("name%d", i) title := fmt.Sprintf(`test foo%d%sbar`, i, v) - query := newQueryByTitle(title) - res, err := index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - if res.TotalHits != 1 { - t.Logf("i: %d, v: %s, title: %s", i, v, title) - } - require.Equal(t, int64(1), res.TotalHits) + indexDocumentsWithTitles(t, index, key, map[string]string{ + name: title, + }) + + checkSearchQuery(t, index, newQueryByTitle(title), []string{name}) // can search for a title with a term character suffix - title = fmt.Sprintf(`foo%d%s`, i, v) - query = newQueryByTitle(title) - res, err = index.Search(context.Background(), nil, query, nil) - require.NoError(t, err) - if res.TotalHits != 1 { - t.Logf("i: %d, v: %s, title: %s\n", i, v, title) - } - - require.Equal(t, int64(1), res.TotalHits) + checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name}) } }) }