diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 5c9f8e86e73..60f255e5489 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -712,10 +712,6 @@ export interface FeatureToggles { */ unifiedStorageSearchSprinkles?: boolean; /** - * Use full n-gram indexing instead of edge n-gram for unified storage search - */ - unifiedStorageUseFullNgram?: boolean; - /** * Pick the dual write mode from database configs */ managedDualWriter?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 2401485935a..67be912cd27 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1227,14 +1227,6 @@ var ( HideFromDocs: true, HideFromAdminPage: true, }, - { - Name: "unifiedStorageUseFullNgram", - Description: "Use full n-gram indexing instead of edge n-gram for unified storage search", - Stage: FeatureStageExperimental, - Owner: grafanaSearchAndStorageSquad, - HideFromDocs: true, - HideFromAdminPage: true, - }, { Name: "managedDualWriter", Description: "Pick the dual write mode from database configs", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 1da8578aa4b..1f24ed369a6 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -160,7 +160,6 @@ useSessionStorageForRedirection,GA,@grafana/identity-access-team,false,false,fal rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false unifiedStorageSearchSprinkles,experimental,@grafana/search-and-storage,false,false,false -unifiedStorageUseFullNgram,experimental,@grafana/search-and-storage,false,false,false managedDualWriter,experimental,@grafana/search-and-storage,false,false,false pluginsSriChecks,GA,@grafana/plugins-platform-backend,false,false,false unifiedStorageBigObjectsSupport,experimental,@grafana/search-and-storage,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 991877fad16..6df7270742a 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -651,10 +651,6 @@ const ( // Enable sprinkles on unified storage search FlagUnifiedStorageSearchSprinkles = "unifiedStorageSearchSprinkles" - // FlagUnifiedStorageUseFullNgram - // Use full n-gram indexing instead of edge n-gram for unified storage search - FlagUnifiedStorageUseFullNgram = "unifiedStorageUseFullNgram" - // FlagManagedDualWriter // Pick the dual write mode from database configs FlagManagedDualWriter = "managedDualWriter" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 7a653f1a15e..e726a825ddb 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -4056,7 +4056,8 @@ "metadata": { "name": "unifiedStorageUseFullNgram", "resourceVersion": "1758820248165", - "creationTimestamp": "2025-09-25T17:10:48Z" + "creationTimestamp": "2025-09-25T17:10:48Z", + "deletionTimestamp": "2025-10-22T12:24:39Z" }, "spec": { "description": "Use full n-gram indexing instead of edge n-gram for unified storage search", diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 0f3ab8b2614..34ac36ecccd 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -73,8 +73,6 @@ type BleveOptions struct { Logger *slog.Logger - UseFullNgram bool - // Minimum time between index updates. IndexMinUpdateInterval time.Duration @@ -97,9 +95,6 @@ type bleveBackend struct { indexMetrics *resource.BleveIndexMetrics - // if true will use ngram instead of edge_ngram for title indexes. See custom_analyzers.go - useFullNgram bool - bgTasksCancel func() bgTasksWg sync.WaitGroup } @@ -148,7 +143,6 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resou opts: opts, ownsIndexFn: ownFn, indexMetrics: indexMetrics, - useFullNgram: opts.UseFullNgram, } ctx, cancel := context.WithCancel(context.Background()) @@ -375,7 +369,7 @@ func (b *bleveBackend) BuildIndex( attribute.String("reason", indexBuildReason), ) - mapper, err := GetBleveMappings(fields, b.useFullNgram) + mapper, err := GetBleveMappings(fields) if err != nil { return nil, err } diff --git a/pkg/storage/unified/search/bleve_integration_test.go b/pkg/storage/unified/search/bleve_integration_test.go index b6d31526385..9ee16a1b559 100644 --- a/pkg/storage/unified/search/bleve_integration_test.go +++ b/pkg/storage/unified/search/bleve_integration_test.go @@ -20,29 +20,6 @@ func TestBleveSearchBackend(t *testing.T) { backend, err := NewBleveBackend(BleveOptions{ Root: tempDir, FileThreshold: 5, - UseFullNgram: false, - }, tracing.NewNoopTracerService(), nil) - require.NoError(t, err) - require.NotNil(t, backend) - - t.Cleanup(backend.Stop) - - return backend - }, &unitest.TestOptions{ - NSPrefix: "bleve-test", - }) -} - -func TestBleveSearchBackendFullNgramEnabled(t *testing.T) { - // Run the search backend test suite - unitest.RunSearchBackendTest(t, func(ctx context.Context) resource.SearchBackend { - tempDir := t.TempDir() - - // Create a new bleve backend - backend, err := NewBleveBackend(BleveOptions{ - Root: tempDir, - FileThreshold: 5, - UseFullNgram: true, }, tracing.NewNoopTracerService(), nil) require.NoError(t, err) require.NotNil(t, backend) @@ -67,31 +44,7 @@ func TestSearchBackendBenchmark(t *testing.T) { // Create a new bleve backend backend, err := NewBleveBackend(BleveOptions{ - Root: tempDir, - UseFullNgram: false, - }, tracing.NewNoopTracerService(), nil) - require.NoError(t, err) - require.NotNil(t, backend) - - t.Cleanup(backend.Stop) - - unitest.BenchmarkSearchBackend(t, backend, opts) -} - -func TestSearchBackendBenchmarkFullNgramEnabled(t *testing.T) { - opts := &unitest.BenchmarkOptions{ - NumResources: 10000, - Concurrency: 1, // For now we only want to test the write throughput - NumNamespaces: 1, - NumGroups: 1, - NumResourceTypes: 1, - } - tempDir := t.TempDir() - - // Create a new bleve backend - backend, err := NewBleveBackend(BleveOptions{ - Root: tempDir, - UseFullNgram: true, + Root: tempDir, }, tracing.NewNoopTracerService(), nil) require.NoError(t, err) require.NotNil(t, backend) diff --git a/pkg/storage/unified/search/bleve_mappings.go b/pkg/storage/unified/search/bleve_mappings.go index 4f443f90bda..9f08c01bedc 100644 --- a/pkg/storage/unified/search/bleve_mappings.go +++ b/pkg/storage/unified/search/bleve_mappings.go @@ -9,10 +9,10 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" ) -func GetBleveMappings(fields resource.SearchableDocumentFields, useFullNgram bool) (mapping.IndexMapping, error) { +func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) { mapper := bleve.NewIndexMapping() - err := RegisterCustomAnalyzers(mapper, useFullNgram) + err := RegisterCustomAnalyzers(mapper) if err != nil { return nil, err } diff --git a/pkg/storage/unified/search/bleve_mappings_test.go b/pkg/storage/unified/search/bleve_mappings_test.go index c3ea737744c..3b8027ee06e 100644 --- a/pkg/storage/unified/search/bleve_mappings_test.go +++ b/pkg/storage/unified/search/bleve_mappings_test.go @@ -13,7 +13,7 @@ import ( ) func TestDocumentMapping(t *testing.T) { - mappings, err := search.GetBleveMappings(nil, false) + mappings, err := search.GetBleveMappings(nil) require.NoError(t, err) data := resource.IndexableDocument{ Title: "title", diff --git a/pkg/storage/unified/search/bleve_performance_test.go b/pkg/storage/unified/search/bleve_performance_test.go index 643e6a70f14..00e23d78259 100644 --- a/pkg/storage/unified/search/bleve_performance_test.go +++ b/pkg/storage/unified/search/bleve_performance_test.go @@ -13,14 +13,14 @@ import ( "github.com/stretchr/testify/require" ) -func setupIndex(b testing.TB, useFullNgram bool) resource.ResourceIndex { +func setupIndex(b testing.TB) resource.ResourceIndex { // size := 1000000 // TODO: 200k documents standard size? size := 200000 // batchSize := 1000 slower 8s (for 200k documents) - 34s (for 1M documents) // batchSize := 10000 // faster 5s (for 200k documents) - 27s (for 1M documents) batchSize := 100000 // fasterer 3.5s (for 200k documents) - 27s (for 1M documents) writer := newTestWriter(size, batchSize) - return newTestDashboardsIndex(b, 1, int64(size), writer, useFullNgram) + return newTestDashboardsIndex(b, 1, int64(size), writer) } const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms) @@ -32,12 +32,7 @@ const verbose = false // changes the the indexer settings can cause unforeseen performance issues ( for example: using wildcard queries ) // this will fail if the stats exceed the "normal" thresholds func BenchmarkBleveQuery(b *testing.B) { - testIndex := setupIndex(b, false) - runBenchmark(b, testIndex) -} - -func BenchmarkBleveQueryFullNgram(b *testing.B) { - testIndex := setupIndex(b, true) + testIndex := setupIndex(b) runBenchmark(b, testIndex) } diff --git a/pkg/storage/unified/search/bleve_search_test.go b/pkg/storage/unified/search/bleve_search_test.go index 476f35985a1..c1b18724d96 100644 --- a/pkg/storage/unified/search/bleve_search_test.go +++ b/pkg/storage/unified/search/bleve_search_test.go @@ -58,189 +58,167 @@ func TestCanSearchByTitle(t *testing.T) { Resource: "dashboards", } - for _, useFullNgram := range []bool{false, true} { - name := "when useFullNgram feature flag is " - if useFullNgram { - name += "enabled" - } else { - name += "disabled" - } - - t.Run(name, func(t *testing.T) { - t.Run("and query is empty, sort documents by title instead of search score", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "bbb", - "name2": "aaa", - }) - - checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"}) - }) - - t.Run("will boost phrase match query over match query results", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "I want to say a hello", - "name2": "we want hello", - }) - - checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"}) - }) - - t.Run("will prioritize matches", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "Asserts Dashboards", - "name2": "New dashboard 10", - }) - - checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"}) - }) - - t.Run("all terms must match", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "Dashboard", - "name2": "New dashboard 10", - }) - - checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"}) - }) - - t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "we want hello pls", - "name2": "we want hello", - }) - - checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"}) - }) - - t.Run("title with numbers will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "A123456", - }) - - // search for prefix of title with mix of chars and numbers - checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"}) - // search for whole title - checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"}) - // case insensive search for partial title - checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"}) - }) - - t.Run("title will match escaped characters", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "what\"s up", - "name2": "what\"s that", - }) - - checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"}) - checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"}) - }) - - t.Run("title search will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", - }) - - // search by entire phrase - checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"}) - - // search for word at start - checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"}) - // search for word larger than ngram max size - checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"}) - // search for word at end - checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"}) - // can search for word substring anchored at start of word (edge ngram) - checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"}) - // can search for multiple, non-consecutive words in title - checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"}) - // can search for multiple, non-consecutive words in title - checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"}) - }) - - t.Run("title search will NOT match documents", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", - "name2": "A0456", - "name3": "mash-A02382-10", - }) - - // word that doesn't exist - checkSearchQuery(t, index, newTestQuery("cats"), nil) - // string shorter than 3 chars (ngram min) - checkSearchQuery(t, index, newTestQuery("ma"), nil) - // substring that doesn't exist - checkSearchQuery(t, index, newTestQuery("A01"), nil) - }) - - t.Run("title search with character will match one document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "foo", - }) - - for i, v := range search.TermCharacters { - name := fmt.Sprintf("name%d", i) - title := fmt.Sprintf(`test foo%d%sbar`, i, v) - indexDocumentsWithTitles(t, index, key, map[string]string{ - name: title, - }) - - checkSearchQuery(t, index, newQueryByTitle(title), []string{name}) - - // can search for a title with a term character suffix - checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name}) - } - }) - - t.Run("title search will ignore terms < 3 characters", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "new dashboard", - "name2": "new dash", - "name3": "new", - }) - - // matches everything - checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"}) - // ignore terms shorter than 3 chars - checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"}) - // include terms shorter that are exactly 3 chars - checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"}) - }) - - name := "title search will%smatch term in the middle/end" - if useFullNgram { - name = fmt.Sprintf(name, " ") - } else { - name = fmt.Sprintf(name, " NOT ") - } - t.Run(name, func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) - indexDocumentsWithTitles(t, index, key, map[string]string{ - "name1": "new dashboard", - "name2": "new dash", - "name3": "somedash", - }) - - if useFullNgram { - checkSearchQuery(t, index, newTestQuery("ash"), []string{"name2", "name3", "name1"}) - checkSearchQuery(t, index, newTestQuery("ome"), []string{"name3"}) - } else { - checkSearchQuery(t, index, newTestQuery("ash"), nil) - checkSearchQuery(t, index, newTestQuery("ome"), nil) - } - }) + t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "bbb", + "name2": "aaa", }) - } + + checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"}) + }) + + t.Run("will boost phrase match query over match query results", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a hello", + "name2": "we want hello", + }) + + checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"}) + }) + + t.Run("will prioritize matches", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "Asserts Dashboards", + "name2": "New dashboard 10", + }) + + checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"}) + }) + + t.Run("all terms must match", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "Dashboard", + "name2": "New dashboard 10", + }) + + checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"}) + }) + + t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "we want hello pls", + "name2": "we want hello", + }) + + checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"}) + }) + + t.Run("title with numbers will match document", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "A123456", + }) + + // search for prefix of title with mix of chars and numbers + checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"}) + // search for whole title + checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"}) + // case insensive search for partial title + checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"}) + }) + + t.Run("title will match escaped characters", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "what\"s up", + "name2": "what\"s that", + }) + + checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"}) + checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"}) + }) + + t.Run("title search will match document", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", + }) + + // search by entire phrase + checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"}) + + // search for word at start + checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"}) + // search for word larger than ngram max size + checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"}) + // search for word at end + checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"}) + // can search for word substring anchored at start of word (edge ngram) + checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"}) + // can search for multiple, non-consecutive words in title + checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"}) + // can search for multiple, non-consecutive words in title + checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"}) + }) + + t.Run("title search will NOT match documents", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", + "name2": "A0456", + "name3": "mash-A02382-10", + }) + + // word that doesn't exist + checkSearchQuery(t, index, newTestQuery("cats"), nil) + // string shorter than 3 chars (ngram min) + checkSearchQuery(t, index, newTestQuery("ma"), nil) + // substring that doesn't exist + checkSearchQuery(t, index, newTestQuery("A01"), nil) + }) + + t.Run("title search with character will match one document", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "foo", + }) + + for i, v := range search.TermCharacters { + name := fmt.Sprintf("name%d", i) + title := fmt.Sprintf(`test foo%d%sbar`, i, v) + indexDocumentsWithTitles(t, index, key, map[string]string{ + name: title, + }) + + checkSearchQuery(t, index, newQueryByTitle(title), []string{name}) + + // can search for a title with a term character suffix + checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name}) + } + }) + + t.Run("title search will ignore terms < 3 characters", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "new dashboard", + "name2": "new dash", + "name3": "new", + }) + + // matches everything + checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"}) + // ignore terms shorter than 3 chars + checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"}) + // include terms shorter that are exactly 3 chars + checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"}) + }) + + t.Run("title search will%smatch term in the middle/end", func(t *testing.T) { + index := newTestDashboardsIndex(t, threshold, 2, noop) + indexDocumentsWithTitles(t, index, key, map[string]string{ + "name1": "new dashboard", + "name2": "new dash", + "name3": "somedash", + }) + + checkSearchQuery(t, index, newTestQuery("ash"), []string{"name2", "name3", "name1"}) + checkSearchQuery(t, index, newTestQuery("ome"), []string{"name3"}) + }) } func newTestQuery(query string) *resourcepb.ResourceSearchRequest { @@ -271,7 +249,7 @@ func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest { } } -func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex { +func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer resource.BuildFn) resource.ResourceIndex { key := &resourcepb.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", @@ -280,7 +258,6 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer re backend, err := search.NewBleveBackend(search.BleveOptions{ Root: t.TempDir(), FileThreshold: threshold, // use in-memory for tests - UseFullNgram: useFullNgram, }, tracing.NewNoopTracerService(), nil) require.NoError(t, err) diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go index 4caa3609f7e..7d81fbe19ee 100644 --- a/pkg/storage/unified/search/bleve_test.go +++ b/pkg/storage/unified/search/bleve_test.go @@ -52,22 +52,6 @@ func TestBleveBackend(t *testing.T) { backend, err := NewBleveBackend(BleveOptions{ Root: tmpdir, FileThreshold: 5, // with more than 5 items we create a file on disk - UseFullNgram: false, - }, tracing.NewNoopTracerService(), nil) - require.NoError(t, err) - t.Cleanup(backend.Stop) - - testBleveBackend(t, backend) -} - -func TestBleveBackendFullNgramEnabled(t *testing.T) { - tmpdir, err := os.MkdirTemp("", "grafana-bleve-test") - require.NoError(t, err) - - backend, err := NewBleveBackend(BleveOptions{ - Root: tmpdir, - FileThreshold: 5, // with more than 5 items we create a file on disk - UseFullNgram: true, }, tracing.NewNoopTracerService(), nil) require.NoError(t, err) t.Cleanup(backend.Stop) @@ -791,7 +775,6 @@ func setupBleveBackend(t *testing.T, options ...setupOption) (*bleveBackend, pro IndexCacheTTL: defaultIndexCacheTTL, Logger: slog.New(logtest.NewNopHandler(t)), BuildVersion: buildVersion, - UseFullNgram: false, } for _, opt := range options { opt(&opts) @@ -1575,7 +1558,6 @@ func TestInvalidBuildVersion(t *testing.T) { opts := BleveOptions{ Root: t.TempDir(), BuildVersion: "invalid", - UseFullNgram: false, } _, err := NewBleveBackend(opts, tracing.NewNoopTracerService(), nil) require.ErrorContains(t, err, "cannot parse build version") diff --git a/pkg/storage/unified/search/custom_analyzers.go b/pkg/storage/unified/search/custom_analyzers.go index 7c0279e4f7f..bc4c6a00532 100644 --- a/pkg/storage/unified/search/custom_analyzers.go +++ b/pkg/storage/unified/search/custom_analyzers.go @@ -2,7 +2,6 @@ package search import ( "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" - "github.com/blevesearch/bleve/v2/analysis/token/edgengram" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/ngram" "github.com/blevesearch/bleve/v2/analysis/token/unique" @@ -14,29 +13,18 @@ const TITLE_ANALYZER = "title_analyzer" const EDGE_NGRAM_MIN_TOKEN = 3.0 const tokenFilterName = "ngram_filter" -func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl, useFullNgram bool) error { - return registerTitleAnalyzer(mapper, useFullNgram) +func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl) error { + return registerTitleAnalyzer(mapper) } // The registerTitleAnalyzer function defines a custom analyzer using edge n-gram or full n-gram -func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl, useFullNgram bool) error { - // The edgengram tokenFilter will create grams anchored to the front of each token. - // For example, the token "hello" will be tokenized into "hel", "hell", "hello". +func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error { + // The ngram tokenFilter will create additional grams in the middle of each token. + // For example, the token "hello" will be tokenized into "hel", "hell", "hello", "ell", "ello", "llo". tokenFilter := map[string]interface{}{ - "type": edgengram.Name, + "type": ngram.Name, "min": EDGE_NGRAM_MIN_TOKEN, "max": 10.0, - "back": edgengram.FRONT, - } - - if useFullNgram { - // The ngram tokenFilter will create additional grams in the middle of each token. - // For example, the token "hello" will be tokenized into "hel", "hell", "hello", "ell", "ello", "llo". - tokenFilter = map[string]interface{}{ - "type": ngram.Name, - "min": EDGE_NGRAM_MIN_TOKEN, - "max": 10.0, - } } err := mapper.AddCustomTokenFilter(tokenFilterName, tokenFilter) if err != nil { diff --git a/pkg/storage/unified/search/options.go b/pkg/storage/unified/search/options.go index 47a24659347..e41fb0f13d4 100644 --- a/pkg/storage/unified/search/options.go +++ b/pkg/storage/unified/search/options.go @@ -45,7 +45,6 @@ func NewSearchOptions( FileThreshold: int64(cfg.IndexFileThreshold), // fewer than X items will use a memory index IndexCacheTTL: cfg.IndexCacheTTL, // How long to keep the index cache in memory BuildVersion: cfg.BuildVersion, - UseFullNgram: features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageUseFullNgram), OwnsIndex: ownsIndexFn, IndexMinUpdateInterval: cfg.IndexMinUpdateInterval, }, tracer, indexMetrics)