unified-storage: add feature flag to use ngram for indexing (#111265)

* unified-storage: add feature flag to use ngram instead of edge-ngram for indexing

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -692,6 +692,10 @@ export interface FeatureToggles {
   */
   unifiedStorageSearchSprinkles?: boolean;
   /**
+  * Use full n-gram indexing instead of edge n-gram for unified storage search
+  */
+  unifiedStorageUseFullNgram?: boolean;
+  /**
   * Pick the dual write mode from database configs
   */
   managedDualWriter?: boolean;

pkg/services/featuremgmt/registry.go

@@ -1193,6 +1193,14 @@ var (
 			HideFromDocs:      true,
 			HideFromAdminPage: true,
 		},
+		{
+			Name:              "unifiedStorageUseFullNgram",
+			Description:       "Use full n-gram indexing instead of edge n-gram for unified storage search",
+			Stage:             FeatureStageExperimental,
+			Owner:             grafanaSearchAndStorageSquad,
+			HideFromDocs:      true,
+			HideFromAdminPage: true,
+		},
 		{
 			Name:              "managedDualWriter",
 			Description:       "Pick the dual write mode from database configs",

pkg/services/featuremgmt/toggles_gen.csv

@@ -155,6 +155,7 @@ useSessionStorageForRedirection,GA,@grafana/identity-access-team,false,false,fal
 rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false
 unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false
 unifiedStorageSearchSprinkles,experimental,@grafana/search-and-storage,false,false,false
+unifiedStorageUseFullNgram,experimental,@grafana/search-and-storage,false,false,false
 managedDualWriter,experimental,@grafana/search-and-storage,false,false,false
 pluginsSriChecks,GA,@grafana/plugins-platform-backend,false,false,false
 unifiedStorageBigObjectsSupport,experimental,@grafana/search-and-storage,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -631,6 +631,10 @@ const (
 	// Enable sprinkles on unified storage search
 	FlagUnifiedStorageSearchSprinkles = "unifiedStorageSearchSprinkles"
 
+	// FlagUnifiedStorageUseFullNgram
+	// Use full n-gram indexing instead of edge n-gram for unified storage search
+	FlagUnifiedStorageUseFullNgram = "unifiedStorageUseFullNgram"
+
 	// FlagManagedDualWriter
 	// Pick the dual write mode from database configs
 	FlagManagedDualWriter = "managedDualWriter"

pkg/storage/unified/search/bleve.go

@@ -77,6 +77,8 @@ type BleveOptions struct {
 	MinBuildVersion *semver.Version // Minimum build version for reusing file-based indexes. Ignored if nil.
 
 	Logger *slog.Logger
+
+	UseFullNgram bool
 }
 
 type bleveBackend struct {
@@ -89,6 +91,9 @@ type bleveBackend struct {
 
 	indexMetrics *resource.BleveIndexMetrics
 
+	// if true will use ngram instead of edge_ngram for title indexes. See custom_analyzers.go
+	useFullNgram bool
+
 	metricsUpdaterCancel func()
 	metricsUpdaterWg     sync.WaitGroup
 }
@@ -130,6 +135,7 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resou
 		cache:        map[resource.NamespacedResource]*bleveIndex{},
 		opts:         opts,
 		indexMetrics: indexMetrics,
+		useFullNgram: opts.UseFullNgram,
 	}
 
 	if be.indexMetrics != nil {
@@ -314,7 +320,7 @@ func (b *bleveBackend) BuildIndex(
 		attribute.String("reason", indexBuildReason),
 	)
 
-	mapper, err := GetBleveMappings(fields)
+	mapper, err := GetBleveMappings(fields, b.useFullNgram)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/unified/search/bleve_integration_test.go

@@ -20,6 +20,29 @@ func TestBleveSearchBackend(t *testing.T) {
 		backend, err := NewBleveBackend(BleveOptions{
 			Root:          tempDir,
 			FileThreshold: 5,
+			UseFullNgram:  false,
+		}, tracing.NewNoopTracerService(), nil)
+		require.NoError(t, err)
+		require.NotNil(t, backend)
+
+		t.Cleanup(backend.closeAllIndexes)
+
+		return backend
+	}, &unitest.TestOptions{
+		NSPrefix: "bleve-test",
+	})
+}
+
+func TestBleveSearchBackendFullNgramEnabled(t *testing.T) {
+	// Run the search backend test suite
+	unitest.RunSearchBackendTest(t, func(ctx context.Context) resource.SearchBackend {
+		tempDir := t.TempDir()
+
+		// Create a new bleve backend
+		backend, err := NewBleveBackend(BleveOptions{
+			Root:          tempDir,
+			FileThreshold: 5,
+			UseFullNgram:  true,
 		}, tracing.NewNoopTracerService(), nil)
 		require.NoError(t, err)
 		require.NotNil(t, backend)
@@ -44,7 +67,31 @@ func TestSearchBackendBenchmark(t *testing.T) {
 
 	// Create a new bleve backend
 	backend, err := NewBleveBackend(BleveOptions{
-		Root: tempDir,
+		Root:         tempDir,
+		UseFullNgram: false,
+	}, tracing.NewNoopTracerService(), nil)
+	require.NoError(t, err)
+	require.NotNil(t, backend)
+
+	t.Cleanup(backend.closeAllIndexes)
+
+	unitest.BenchmarkSearchBackend(t, backend, opts)
+}
+
+func TestSearchBackendBenchmarkFullNgramEnabled(t *testing.T) {
+	opts := &unitest.BenchmarkOptions{
+		NumResources:     10000,
+		Concurrency:      1, // For now we only want to test the write throughput
+		NumNamespaces:    1,
+		NumGroups:        1,
+		NumResourceTypes: 1,
+	}
+	tempDir := t.TempDir()
+
+	// Create a new bleve backend
+	backend, err := NewBleveBackend(BleveOptions{
+		Root:         tempDir,
+		UseFullNgram: true,
 	}, tracing.NewNoopTracerService(), nil)
 	require.NoError(t, err)
 	require.NotNil(t, backend)

pkg/storage/unified/search/bleve_mappings.go

@@ -9,10 +9,10 @@ import (
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
 
-func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
+func GetBleveMappings(fields resource.SearchableDocumentFields, useFullNgram bool) (mapping.IndexMapping, error) {
 	mapper := bleve.NewIndexMapping()
 
-	err := RegisterCustomAnalyzers(mapper)
+	err := RegisterCustomAnalyzers(mapper, useFullNgram)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/unified/search/bleve_mappings_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestDocumentMapping(t *testing.T) {
-	mappings, err := search.GetBleveMappings(nil)
+	mappings, err := search.GetBleveMappings(nil, false)
 	require.NoError(t, err)
 	data := resource.IndexableDocument{
 		Title:       "title",

pkg/storage/unified/search/bleve_performance_test.go

@@ -13,14 +13,14 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func setupIndex(b testing.TB) resource.ResourceIndex {
+func setupIndex(b testing.TB, useFullNgram bool) resource.ResourceIndex {
 	// size := 1000000  // TODO: 200k documents standard size?
 	size := 200000
 	// batchSize := 1000 slower 8s (for 200k documents) - 34s (for 1M documents)
 	// batchSize := 10000 // faster 5s  (for 200k documents) - 27s (for 1M documents)
 	batchSize := 100000 // fasterer 3.5s  (for 200k documents) - 27s  (for 1M documents)
 	writer := newTestWriter(size, batchSize)
-	return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer)
+	return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer, useFullNgram)
 }
 
 const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms)
@@ -32,12 +32,20 @@ const verbose = false
 // changes the the indexer settings can cause unforeseen performance issues ( for example: using wildcard queries )
 // this will fail if the stats exceed the "normal" thresholds
 func BenchmarkBleveQuery(b *testing.B) {
+	testIndex := setupIndex(b, false)
+	runBenchmark(b, testIndex)
+}
+
+func BenchmarkBleveQueryFullNgram(b *testing.B) {
+	testIndex := setupIndex(b, true)
+	runBenchmark(b, testIndex)
+}
+
+func runBenchmark(b *testing.B, testIndex resource.ResourceIndex) {
 	var memStatsStart runtime.MemStats
 	var memStatsAfterIndex runtime.MemStats
 	runtime.ReadMemStats(&memStatsStart)
 
-	testIndex := setupIndex(b)
-
 	runtime.ReadMemStats(&memStatsAfterIndex)
 
 	allocDiff := memStatsAfterIndex.Alloc - memStatsStart.Alloc

pkg/storage/unified/search/bleve_search_test.go

@@ -58,155 +58,189 @@ func TestCanSearchByTitle(t *testing.T) {
 		Resource:  "dashboards",
 	}
 
-	t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) {
+	for _, useFullNgram := range []bool{false, true} {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
+		name := "when useFullNgram feature flag is "
-		indexDocumentsWithTitles(t, index, key, map[string]string{
+		if useFullNgram {
-			"name1": "bbb",
+			name += "enabled"
-			"name2": "aaa",
+		} else {
-		})
+			name += "disabled"
+		}
 
-		checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"})
+		t.Run(name, func(t *testing.T) {
-	})
+			t.Run("and query is empty, sort documents by title instead of search score", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "bbb",
+					"name2": "aaa",
+				})
 
-	t.Run("will boost phrase match query over match query results", func(t *testing.T) {
+				checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"})
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "I want to say a hello",
-			"name2": "we want hello",
-		})
-
-		checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"})
-	})
-
-	t.Run("will prioritize matches", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "Asserts Dashboards",
-			"name2": "New dashboard 10",
-		})
-
-		checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"})
-	})
-
-	t.Run("all terms must match", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "Dashboard",
-			"name2": "New dashboard 10",
-		})
-
-		checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"})
-	})
-
-	t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "we want hello pls",
-			"name2": "we want hello",
-		})
-
-		checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"})
-	})
-
-	t.Run("title with numbers will match document", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "A123456",
-		})
-
-		// search for prefix of title with mix of chars and numbers
-		checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"})
-		// search for whole title
-		checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"})
-		// case insensive search for partial title
-		checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"})
-	})
-
-	t.Run("title will match escaped characters", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "what\"s up",
-			"name2": "what\"s that",
-		})
-
-		checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"})
-		checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"})
-	})
-
-	t.Run("title search will match document", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
-		})
-
-		// search by entire phrase
-		checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"})
-
-		// search for word at start
-		checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"})
-		// search for word larger than ngram max size
-		checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"})
-		// search for word at end
-		checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"})
-		// can search for word substring anchored at start of word (edge ngram)
-		checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"})
-		// can search for multiple, non-consecutive words in title
-		checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"})
-		// can search for multiple, non-consecutive words in title
-		checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"})
-	})
-
-	t.Run("title search will NOT match documents", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
-			"name2": "A0456",
-			"name3": "mash-A02382-10",
-		})
-
-		// word that doesn't exist
-		checkSearchQuery(t, index, newTestQuery("cats"), nil)
-		// string shorter than 3 chars (ngam min)
-		checkSearchQuery(t, index, newTestQuery("ma"), nil)
-		// substring that doesn't exist
-		checkSearchQuery(t, index, newTestQuery("A01"), nil)
-	})
-
-	t.Run("title search with character will match one document", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "foo",
-		})
-
-		for i, v := range search.TermCharacters {
-			name := fmt.Sprintf("name%d", i)
-			title := fmt.Sprintf(`test foo%d%sbar`, i, v)
-			indexDocumentsWithTitles(t, index, key, map[string]string{
-				name: title,
 			})
 
-			checkSearchQuery(t, index, newQueryByTitle(title), []string{name})
+			t.Run("will boost phrase match query over match query results", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "I want to say a hello",
+					"name2": "we want hello",
+				})
 
-			// can search for a title with a term character suffix
+				checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"})
-			checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name})
+			})
-		}
-	})
 
-	t.Run("title search will ignore terms < 3 characters", func(t *testing.T) {
+			t.Run("will prioritize matches", func(t *testing.T) {
-		index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
-		indexDocumentsWithTitles(t, index, key, map[string]string{
+				indexDocumentsWithTitles(t, index, key, map[string]string{
-			"name1": "new dashboard",
+					"name1": "Asserts Dashboards",
-			"name2": "new dash",
+					"name2": "New dashboard 10",
-			"name3": "new",
+				})
+
+				checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"})
+			})
+
+			t.Run("all terms must match", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "Dashboard",
+					"name2": "New dashboard 10",
+				})
+
+				checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"})
+			})
+
+			t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "we want hello pls",
+					"name2": "we want hello",
+				})
+
+				checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"})
+			})
+
+			t.Run("title with numbers will match document", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "A123456",
+				})
+
+				// search for prefix of title with mix of chars and numbers
+				checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"})
+				// search for whole title
+				checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"})
+				// case insensive search for partial title
+				checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"})
+			})
+
+			t.Run("title will match escaped characters", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "what\"s up",
+					"name2": "what\"s that",
+				})
+
+				checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"})
+				checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"})
+			})
+
+			t.Run("title search will match document", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
+				})
+
+				// search by entire phrase
+				checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"})
+
+				// search for word at start
+				checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"})
+				// search for word larger than ngram max size
+				checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"})
+				// search for word at end
+				checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"})
+				// can search for word substring anchored at start of word (edge ngram)
+				checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"})
+				// can search for multiple, non-consecutive words in title
+				checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"})
+				// can search for multiple, non-consecutive words in title
+				checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"})
+			})
+
+			t.Run("title search will NOT match documents", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
+					"name2": "A0456",
+					"name3": "mash-A02382-10",
+				})
+
+				// word that doesn't exist
+				checkSearchQuery(t, index, newTestQuery("cats"), nil)
+				// string shorter than 3 chars (ngram min)
+				checkSearchQuery(t, index, newTestQuery("ma"), nil)
+				// substring that doesn't exist
+				checkSearchQuery(t, index, newTestQuery("A01"), nil)
+			})
+
+			t.Run("title search with character will match one document", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "foo",
+				})
+
+				for i, v := range search.TermCharacters {
+					name := fmt.Sprintf("name%d", i)
+					title := fmt.Sprintf(`test foo%d%sbar`, i, v)
+					indexDocumentsWithTitles(t, index, key, map[string]string{
+						name: title,
+					})
+
+					checkSearchQuery(t, index, newQueryByTitle(title), []string{name})
+
+					// can search for a title with a term character suffix
+					checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name})
+				}
+			})
+
+			t.Run("title search will ignore terms < 3 characters", func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "new dashboard",
+					"name2": "new dash",
+					"name3": "new",
+				})
+
+				// matches everything
+				checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"})
+				// ignore terms shorter than 3 chars
+				checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"})
+				// include terms shorter that are exactly 3 chars
+				checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"})
+			})
+
+			name := "title search will%smatch term in the middle/end"
+			if useFullNgram {
+				name = fmt.Sprintf(name, " ")
+			} else {
+				name = fmt.Sprintf(name, " NOT ")
+			}
+			t.Run(name, func(t *testing.T) {
+				index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
+				indexDocumentsWithTitles(t, index, key, map[string]string{
+					"name1": "new dashboard",
+					"name2": "new dash",
+					"name3": "somedash",
+				})
+
+				if useFullNgram {
+					checkSearchQuery(t, index, newTestQuery("ash"), []string{"name2", "name3", "name1"})
+					checkSearchQuery(t, index, newTestQuery("ome"), []string{"name3"})
+				} else {
+					checkSearchQuery(t, index, newTestQuery("ash"), nil)
+					checkSearchQuery(t, index, newTestQuery("ome"), nil)
+				}
+			})
 		})
-
+	}
-		// matches everything
-		checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"})
-		// ignore terms shorter than 3 chars
-		checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"})
-		// include terms shorter that are exactly 3 chars
-		checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"})
-	})
 }
 
 func newTestQuery(query string) *resourcepb.ResourceSearchRequest {
@@ -237,7 +271,7 @@ func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest {
 	}
 }
 
-func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn) resource.ResourceIndex {
+func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex {
 	key := &resourcepb.ResourceKey{
 		Namespace: "default",
 		Group:     "dashboard.grafana.app",
@@ -247,6 +281,7 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize
 		Root:          t.TempDir(),
 		FileThreshold: threshold, // use in-memory for tests
 		BatchSize:     int(batchSize),
+		UseFullNgram:  useFullNgram,
 	}, tracing.NewNoopTracerService(), nil)
 	require.NoError(t, err)
 

pkg/storage/unified/search/bleve_test.go

@@ -46,6 +46,34 @@ func TestMain(m *testing.M) {
 }
 
 func TestBleveBackend(t *testing.T) {
+	tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
+	require.NoError(t, err)
+
+	backend, err := NewBleveBackend(BleveOptions{
+		Root:          tmpdir,
+		FileThreshold: 5, // with more than 5 items we create a file on disk
+		UseFullNgram:  false,
+	}, tracing.NewNoopTracerService(), nil)
+	require.NoError(t, err)
+
+	testBleveBackend(t, backend)
+}
+
+func TestBleveBackendFullNgramEnabled(t *testing.T) {
+	tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
+	require.NoError(t, err)
+
+	backend, err := NewBleveBackend(BleveOptions{
+		Root:          tmpdir,
+		FileThreshold: 5, // with more than 5 items we create a file on disk
+		UseFullNgram:  true,
+	}, tracing.NewNoopTracerService(), nil)
+	require.NoError(t, err)
+
+	testBleveBackend(t, backend)
+}
+
+func testBleveBackend(t *testing.T, backend *bleveBackend) {
 	dashboardskey := &resourcepb.ResourceKey{
 		Namespace: "default",
 		Group:     "dashboard.grafana.app",
@@ -56,14 +84,6 @@ func TestBleveBackend(t *testing.T) {
 		Group:     "folder.grafana.app",
 		Resource:  "folders",
 	}
-	tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
-	require.NoError(t, err)
-
-	backend, err := NewBleveBackend(BleveOptions{
-		Root:          tmpdir,
-		FileThreshold: 5, // with more than 5 items we create a file on disk
-	}, tracing.NewNoopTracerService(), nil)
-	require.NoError(t, err)
 
 	t.Cleanup(backend.Stop)
 
@@ -771,6 +791,7 @@ func setupBleveBackend(t *testing.T, options ...setupOption) (*bleveBackend, pro
 		IndexCacheTTL: defaultIndexCacheTTL,
 		Logger:        slog.New(logtest.NewNopHandler(t)),
 		BuildVersion:  buildVersion,
+		UseFullNgram:  false,
 	}
 	for _, opt := range options {
 		opt(&opts)
@@ -1467,6 +1488,7 @@ func TestInvalidBuildVersion(t *testing.T) {
 	opts := BleveOptions{
 		Root:         t.TempDir(),
 		BuildVersion: "invalid",
+		UseFullNgram: false,
 	}
 	_, err := NewBleveBackend(opts, tracing.NewNoopTracerService(), nil)
 	require.ErrorContains(t, err, "cannot parse build version")

pkg/storage/unified/search/custom_analyzers.go

@@ -4,6 +4,7 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/v2/analysis/token/edgengram"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/token/ngram"
 	"github.com/blevesearch/bleve/v2/analysis/token/unique"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace"
 	"github.com/blevesearch/bleve/v2/mapping"
@@ -11,23 +12,33 @@ import (
 
 const TITLE_ANALYZER = "title_analyzer"
 const EDGE_NGRAM_MIN_TOKEN = 3.0
+const tokenFilterName = "ngram_filter"
 
-func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl) error {
+func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl, useFullNgram bool) error {
-	return registerTitleAnalyzer(mapper)
+	return registerTitleAnalyzer(mapper, useFullNgram)
 }
 
-// The registerTitleAnalyzer function defines a custom analyzer for the title field.
+// The registerTitleAnalyzer function defines a custom analyzer using edge n-gram or full n-gram
-// The edgeNgramTokenFilter will create n-grams anchored to the front of each token.
+func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl, useFullNgram bool) error {
-// For example, the token "hello" will be tokenized into "hel", "hell", "hello".
+	// The edgengram tokenFilter will create grams anchored to the front of each token.
-func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error {
+	// For example, the token "hello" will be tokenized into "hel", "hell", "hello".
-	// Define an N-Gram tokenizer (for substring search)
+	tokenFilter := map[string]interface{}{
-	edgeNgramTokenFilter := map[string]interface{}{
 		"type": edgengram.Name,
 		"min":  EDGE_NGRAM_MIN_TOKEN,
 		"max":  10.0,
 		"back": edgengram.FRONT,
 	}
-	err := mapper.AddCustomTokenFilter("edge_ngram_filter", edgeNgramTokenFilter)
+
+	if useFullNgram {
+		// The ngram tokenFilter will create additional grams in the middle of each token.
+		// For example, the token "hello" will be tokenized into "hel", "hell", "hello", "ell", "ello", "llo".
+		tokenFilter = map[string]interface{}{
+			"type": ngram.Name,
+			"min":  EDGE_NGRAM_MIN_TOKEN,
+			"max":  10.0,
+		}
+	}
+	err := mapper.AddCustomTokenFilter(tokenFilterName, tokenFilter)
 	if err != nil {
 		return err
 	}
@@ -36,7 +47,7 @@ func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error {
 	ngramAnalyzer := map[string]interface{}{
 		"type":          custom.Name,
 		"tokenizer":     whitespace.Name,
-		"token_filters": []string{"edge_ngram_filter", lowercase.Name, unique.Name},
+		"token_filters": []string{tokenFilterName, lowercase.Name, unique.Name},
 		//"char_filters":  //TODO IF NEEDED
 	}
 

pkg/storage/unified/search/options.go

@@ -43,6 +43,7 @@ func NewSearchOptions(features featuremgmt.FeatureToggles, cfg *setting.Cfg, tra
 			BuildVersion:    cfg.BuildVersion,
 			MaxFileIndexAge: cfg.MaxFileIndexAge,
 			MinBuildVersion: minVersion,
+			UseFullNgram:    features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageUseFullNgram),
 		}, tracer, indexMetrics)
 
 		if err != nil {

pkg/storage/unified/sql/test/integration_test.go

@@ -110,8 +110,6 @@ func TestIntegrationSearchAndStorage(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, search)
 
-	t.Cleanup(search.Stop)
-
 	// Create a new resource backend
 	storage := newTestBackend(t, false, 0)
 	require.NotNil(t, storage)