unified-storage search: return results matching all search terms (#110672)

* Extract common indexing code.

* Extract common search code.

* Match all terms in the search query.

pkg/storage/unified/search/bleve.go

@@ -1072,16 +1072,18 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R
 		// Query 1: Match the exact query string
 		queryExact := bleve.NewMatchQuery(req.Query)
 		queryExact.SetBoost(10.0)
-		queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token
+		queryExact.Analyzer = keyword.Name                // don't analyze the query input - treat it as a single token
+		queryExact.Operator = query.MatchQueryOperatorAnd // This doesn't make a difference for keyword analyzer, we add it just to be explicit.
 
 		// Query 2: Phrase query with standard analyzer
 		queryPhrase := bleve.NewMatchPhraseQuery(req.Query)
-		queryExact.SetBoost(5.0)
+		queryPhrase.SetBoost(5.0)
 		queryPhrase.Analyzer = standard.Name
 
 		// Query 3: Match query with standard analyzer
 		queryAnalyzed := bleve.NewMatchQuery(req.Query)
 		queryAnalyzed.Analyzer = standard.Name
+		queryAnalyzed.Operator = query.MatchQueryOperatorAnd // Make sure all terms from the query are matched
 
 		// At least one of the queries must match
 		searchQuery := bleve.NewDisjunctionQuery(queryExact, queryAnalyzed, queryPhrase)

pkg/storage/unified/search/bleve_search_test.go

@@ -21,8 +21,39 @@ import (
 
 const threshold = 9999
 
+func indexDocumentsWithTitles(t *testing.T, index resource.ResourceIndex, key resource.NamespacedResource, docsWithTitles map[string]string) {
+	items := make([]*resource.BulkIndexItem, 0, len(docsWithTitles))
+	for name, title := range docsWithTitles {
+		items = append(items, &resource.BulkIndexItem{
+			Action: resource.ActionIndex,
+			Doc: &resource.IndexableDocument{
+				RV:   1,
+				Name: name,
+				Key: &resourcepb.ResourceKey{
+					Name:      name,
+					Namespace: key.Namespace,
+					Group:     key.Group,
+					Resource:  key.Resource,
+				},
+				Title: title,
+			},
+		})
+	}
+	req := &resource.BulkIndexRequest{Items: items}
+	require.NoError(t, index.BulkIndex(req))
+}
+
+func checkSearchQuery(t *testing.T, index resource.ResourceIndex, query *resourcepb.ResourceSearchRequest, orderedExpectedNames []string) {
+	res, err := index.Search(context.Background(), nil, query, nil)
+	require.NoError(t, err)
+	require.Equal(t, int64(len(orderedExpectedNames)), res.TotalHits)
+	for ix, name := range orderedExpectedNames {
+		require.Equal(t, name, res.Results.Rows[ix].Key.Name)
+	}
+}
+
 func TestCanSearchByTitle(t *testing.T) {
-	key := &resourcepb.ResourceKey{
+	key := resource.NamespacedResource{
 		Namespace: "default",
 		Group:     "dashboard.grafana.app",
 		Resource:  "dashboards",
@@ -30,463 +61,135 @@ func TestCanSearchByTitle(t *testing.T) {
 
 	t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "bbb",
-				{
+			"name2": "aaa",
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name1",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "bbb",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "aaa",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
-		// search for phrase
+		checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"})
-		query := newTestQuery("")
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(2), res.TotalHits)
-		require.Equal(t, "name2", res.Results.Rows[0].Key.Name)
 	})
 
 	t.Run("will boost phrase match query over match query results", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "I want to say a hello",
-				{
+			"name2": "we want hello",
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name1",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "I want to say a hello",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "we want hello",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
-		// search for phrase
+		checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"})
-		query := newTestQuery("want hello")
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(2), res.TotalHits)
-		require.Equal(t, "name2", res.Results.Rows[0].Key.Name)
 	})
 
 	t.Run("will prioritize matches", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "Asserts Dashboards",
-				{
+			"name2": "New dashboard 10",
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name1",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "Asserts Dashboards",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "New dashboard 10",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
-		query := newTestQuery("New dash")
+		checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"})
-		res, err := index.Search(context.Background(), nil, query, nil)
+	})
-		require.NoError(t, err)
+
-		require.Equal(t, int64(2), res.TotalHits)
+	t.Run("all terms must match", func(t *testing.T) {
-		require.Equal(t, "name2", res.Results.Rows[0].Key.Name)
+		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
+		indexDocumentsWithTitles(t, index, key, map[string]string{
+			"name1": "Dashboard",
+			"name2": "New dashboard 10",
+		})
+
+		checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"})
 	})
 
 	t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "we want hello pls",
-				{
+			"name2": "we want hello",
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name1",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "we want hello pls",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "we want hello",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
-		// search for exact match
+		checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"})
-		query := newTestQuery("we want hello")
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(2), res.TotalHits)
-		require.Equal(t, "name2", res.Results.Rows[0].Key.Name)
 	})
 
 	t.Run("title with numbers will match document", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "A123456",
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "aaa",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "A123456",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
 		// search for prefix of title with mix of chars and numbers
-		query := newQueryByTitle("A12")
+		checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"})
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// search for whole title
-		query = newQueryByTitle("A123456")
+		checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// case insensive search for partial title
-		query = newQueryByTitle("a1234")
+		checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
 	})
 
 	t.Run("title will match escaped characters", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "what\"s up",
-				{
+			"name2": "what\"s that",
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "aaa",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "what\"s up",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   2,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "what\"s that",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
-		query := newQueryByTitle("what\"s up")
+		checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"})
-		res, err := index.Search(context.Background(), nil, query, nil)
+		checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"})
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
-		query = newQueryByTitle("what\"s")
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(2), res.TotalHits)
 	})
 
 	t.Run("title search will match document", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "aaa",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "I want to say a wonderfully Hello to the WORLD! Hello-world",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
 		// search by entire phrase
-		query := newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world")
+		checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"})
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
 
 		// search for word at start
-		query = newTestQuery("hello")
+		checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// search for word larger than ngram max size
-		query = newQueryByTitle("wonderfully")
+		checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// search for word at end
-		query = newQueryByTitle("world")
+		checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// can search for word substring anchored at start of word (edge ngram)
-		query = newQueryByTitle("worl")
+		checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
-
 		// can search for multiple, non-consecutive words in title
-		query = newQueryByTitle("hello world")
+		checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"})
-		res, err = index.Search(context.Background(), nil, query, nil)
+		// can search for multiple, non-consecutive words in title
-		require.NoError(t, err)
+		checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"})
-		require.Equal(t, int64(1), res.TotalHits)
-
-		// can search for a term with a hyphen
-		query = newQueryByTitle("hello-world")
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(1), res.TotalHits)
 	})
 
 	t.Run("title search will NOT match documents", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
-				{
+			"name2": "A0456",
-					Action: resource.ActionIndex,
+			"name3": "mash-A02382-10",
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name1",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "I want to say a wonderful Hello to the WORLD! Hello-world",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name2",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name2",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "A0456",
-					},
-				},
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name3",
-						Key: &resourcepb.ResourceKey{
-							Name:      "name3",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "mash-A02382-10",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
 		// word that doesn't exist
-		query := newQueryByTitle("cats")
+		checkSearchQuery(t, index, newTestQuery("cats"), nil)
-		res, err := index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(0), res.TotalHits)
-
 		// string shorter than 3 chars (ngam min)
-		query = newQueryByTitle("ma")
+		checkSearchQuery(t, index, newTestQuery("ma"), nil)
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(0), res.TotalHits)
-
 		// substring that doesn't exist
-		query = newQueryByTitle("A01")
+		checkSearchQuery(t, index, newTestQuery("A01"), nil)
-		res, err = index.Search(context.Background(), nil, query, nil)
-		require.NoError(t, err)
-		require.Equal(t, int64(0), res.TotalHits)
 	})
 
 	t.Run("title search with character will match one document", func(t *testing.T) {
 		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		err := index.BulkIndex(&resource.BulkIndexRequest{
+		indexDocumentsWithTitles(t, index, key, map[string]string{
-			Items: []*resource.BulkIndexItem{
+			"name1": "foo",
-				{
-					Action: resource.ActionIndex,
-					Doc: &resource.IndexableDocument{
-						RV:   1,
-						Name: "name1",
-						Key: &resourcepb.ResourceKey{
-							Name:      "aaa",
-							Namespace: key.Namespace,
-							Group:     key.Group,
-							Resource:  key.Resource,
-						},
-						Title: "foo",
-					},
-				},
-			},
 		})
-		require.NoError(t, err)
 
 		for i, v := range search.TermCharacters {
-			err = index.BulkIndex(&resource.BulkIndexRequest{
+			name := fmt.Sprintf("name%d", i)
-				Items: []*resource.BulkIndexItem{
-					{
-						Action: resource.ActionIndex,
-						Doc: &resource.IndexableDocument{
-							RV:   int64(i),
-							Name: fmt.Sprintf("name%d", i),
-							Key: &resourcepb.ResourceKey{
-								Name:      fmt.Sprintf("name%d", i),
-								Namespace: key.Namespace,
-								Group:     key.Group,
-								Resource:  key.Resource,
-							},
-							Title: fmt.Sprintf(`test foo%d%sbar`, i, v),
-						},
-					},
-				},
-			})
-			require.NoError(t, err)
-
 			title := fmt.Sprintf(`test foo%d%sbar`, i, v)
-			query := newQueryByTitle(title)
+			indexDocumentsWithTitles(t, index, key, map[string]string{
-			res, err := index.Search(context.Background(), nil, query, nil)
+				name: title,
-			require.NoError(t, err)
+			})
-			if res.TotalHits != 1 {
+
-				t.Logf("i: %d, v: %s, title: %s", i, v, title)
+			checkSearchQuery(t, index, newQueryByTitle(title), []string{name})
-			}
-			require.Equal(t, int64(1), res.TotalHits)
 
 			// can search for a title with a term character suffix
-			title = fmt.Sprintf(`foo%d%s`, i, v)
+			checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name})
-			query = newQueryByTitle(title)
-			res, err = index.Search(context.Background(), nil, query, nil)
-			require.NoError(t, err)
-			if res.TotalHits != 1 {
-				t.Logf("i: %d, v: %s, title: %s\n", i, v, title)
-			}
-
-			require.Equal(t, int64(1), res.TotalHits)
 		}
 	})
 }