package search_test import ( "context" "fmt" "log" "testing" "github.com/blevesearch/bleve/v2" index "github.com/blevesearch/bleve_index_api" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/services/store/kind/dashboard" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/storage/unified/search/builders" ) const threshold = 9999 func indexDocumentsWithTitles(t *testing.T, index resource.ResourceIndex, key resource.NamespacedResource, docsWithTitles map[string]string) { items := make([]*resource.BulkIndexItem, 0, len(docsWithTitles)) for name, title := range docsWithTitles { items = append(items, &resource.BulkIndexItem{ Action: resource.ActionIndex, Doc: &resource.IndexableDocument{ RV: 1, Name: name, Key: &resourcepb.ResourceKey{ Name: name, Namespace: key.Namespace, Group: key.Group, Resource: key.Resource, }, Title: title, }, }) } req := &resource.BulkIndexRequest{Items: items} require.NoError(t, index.BulkIndex(req)) } func checkSearchQuery(t *testing.T, index resource.ResourceIndex, query *resourcepb.ResourceSearchRequest, orderedExpectedNames []string) { res, err := index.Search(context.Background(), nil, query, nil, nil) require.NoError(t, err) require.Equal(t, int64(len(orderedExpectedNames)), res.TotalHits) for ix, name := range orderedExpectedNames { require.Equal(t, name, res.Results.Rows[ix].Key.Name) } } func TestCanSearchByTitle(t *testing.T) { key := resource.NamespacedResource{ Namespace: "default", Group: "dashboard.grafana.app", Resource: "dashboards", } t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "bbb", "name2": "aaa", }) checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"}) }) t.Run("will boost phrase match query over match query results", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a hello", "name2": "we want hello", }) checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"}) }) t.Run("will prioritize matches", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "Asserts Dashboards", "name2": "New dashboard 10", }) checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"}) }) t.Run("all terms must match", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "Dashboard", "name2": "New dashboard 10", }) checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"}) }) t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "we want hello pls", "name2": "we want hello", }) checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"}) }) t.Run("title with numbers will match document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "A123456", }) // search for prefix of title with mix of chars and numbers checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"}) // search for whole title checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"}) // case insensive search for partial title checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"}) }) t.Run("title will match escaped characters", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "what\"s up", "name2": "what\"s that", }) checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"}) checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"}) }) t.Run("title search will match document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", }) // search by entire phrase checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"}) // search for word at start checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"}) // search for word larger than ngram max size checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"}) // search for word at end checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"}) // can search for word substring anchored at start of word (edge ngram) checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"}) // can search for multiple, non-consecutive words in title checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"}) // can search for multiple, non-consecutive words in title checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"}) }) t.Run("title search will NOT match documents", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", "name2": "A0456", "name3": "mash-A02382-10", }) // word that doesn't exist checkSearchQuery(t, index, newTestQuery("cats"), nil) // string shorter than 3 chars (ngram min) checkSearchQuery(t, index, newTestQuery("ma"), nil) // substring that doesn't exist checkSearchQuery(t, index, newTestQuery("A01"), nil) }) t.Run("title search with character will match one document", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "foo", }) for i, v := range search.TermCharacters { name := fmt.Sprintf("name%d", i) title := fmt.Sprintf(`test foo%d%sbar`, i, v) indexDocumentsWithTitles(t, index, key, map[string]string{ name: title, }) checkSearchQuery(t, index, newQueryByTitle(title), []string{name}) // can search for a title with a term character suffix checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name}) } }) t.Run("title search will ignore terms < 3 characters", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "new dashboard", "name2": "new dash", "name3": "new", }) // matches everything checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"}) // ignore terms shorter than 3 chars checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"}) // include terms shorter that are exactly 3 chars checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"}) }) t.Run("title search will%smatch term in the middle/end", func(t *testing.T) { index := newTestDashboardsIndex(t, threshold, 2, noop) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "new dashboard", "name2": "new dash", "name3": "somedash", }) checkSearchQuery(t, index, newTestQuery("ash"), []string{"name2", "name3", "name1"}) checkSearchQuery(t, index, newTestQuery("ome"), []string{"name3"}) }) } func newTestQuery(query string) *resourcepb.ResourceSearchRequest { return &resourcepb.ResourceSearchRequest{ Options: &resourcepb.ListOptions{ Key: &resourcepb.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", Resource: "dashboards", }, }, Limit: 100000, Query: query, } } func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest { return &resourcepb.ResourceSearchRequest{ Options: &resourcepb.ListOptions{ Key: &resourcepb.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", Resource: "dashboards", }, Fields: []*resourcepb.Requirement{{Key: "title", Operator: "=", Values: []string{query}}}, }, Limit: 100000, } } func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer resource.BuildFn) resource.ResourceIndex { key := &resourcepb.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", Resource: "dashboards", } backend, err := search.NewBleveBackend(search.BleveOptions{ Root: t.TempDir(), FileThreshold: threshold, // use in-memory for tests ScoringModel: index.BM25Scoring, }, nil) require.NoError(t, err) t.Cleanup(backend.Stop) ctx := identity.WithRequester(context.Background(), &user.SignedInUser{Namespace: "ns"}) info, err := builders.DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { return &builders.DashboardDocumentBuilder{ Namespace: namespace, Blob: blob, Stats: make(map[string]map[string]int64), // empty stats DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{}}), }, nil }) require.NoError(t, err) index, err := backend.BuildIndex(ctx, resource.NamespacedResource{ Namespace: key.Namespace, Group: key.Group, Resource: key.Resource, }, size, info.Fields, "test", writer, nil, false) require.NoError(t, err) return index } var noop resource.BuildFn = func(index resource.ResourceIndex) (int64, error) { return 0, nil } // helper to check which tokens are generated by an analyzer // nolint:unused func debugAnalyzer(index bleve.Index, analyzerName string, text string) { // Get the analyzer (default: "standard") analyzer := index.Mapping().AnalyzerNamed(analyzerName) if analyzer == nil { log.Fatal("Analyzer not found") } // Analyze text to see generated tokens analysisResult := analyzer.Analyze([]byte(text)) // Print tokens fmt.Println("Generated tokens for analyzer:", analyzerName) for _, token := range analysisResult { fmt.Println(string(token.Term)) } } // helper to check which terms are indexed for a field // nolint:unused func debugIndexedTerms(index bleve.Index, field string) { // Check what terms exist for the title field fieldTerms, err := index.FieldDict(field) if err != nil { log.Fatal(err) } for { term, err := fieldTerms.Next() if err != nil { break } if term != nil { fmt.Println(term.Term) } } }