public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unified-storage: add feature flag to use ngram for indexing (#111265)

Browse Source 
* unified-storage: add feature flag to use ngram instead of edge-ngram for indexing
This commit is contained in:
Will Assis
2025-09-24 10:36:50 +02:00
committed by GitHub
parent 2669e0a770
commit 33ff6dbb9e
14 changed files with 317 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/types/featureToggles.gen.ts
+4
View File
@@ -692,6 +692,10 @@ export interface FeatureToggles {
*/
unifiedStorageSearchSprinkles?: boolean;
/**
* Use full n-gram indexing instead of edge n-gram for unified storage search
*/
unifiedStorageUseFullNgram?: boolean;
/**
* Pick the dual write mode from database configs
*/
managedDualWriter?: boolean;
pkg/services/featuremgmt/registry.go
+8
View File
@@ -1193,6 +1193,14 @@ var (
HideFromDocs: true,
HideFromAdminPage: true,
},
{
Name: "unifiedStorageUseFullNgram",
Description: "Use full n-gram indexing instead of edge n-gram for unified storage search",
Stage: FeatureStageExperimental,
Owner: grafanaSearchAndStorageSquad,
HideFromDocs: true,
HideFromAdminPage: true,
},
{
Name: "managedDualWriter",
Description: "Pick the dual write mode from database configs",
pkg/services/featuremgmt/toggles_gen.csv
+1
View File
@@ -155,6 +155,7 @@ useSessionStorageForRedirection,GA,@grafana/identity-access-team,false,false,fal
rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false
unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false
unifiedStorageSearchSprinkles,experimental,@grafana/search-and-storage,false,false,false
unifiedStorageUseFullNgram,experimental,@grafana/search-and-storage,false,false,false
managedDualWriter,experimental,@grafana/search-and-storage,false,false,false
pluginsSriChecks,GA,@grafana/plugins-platform-backend,false,false,false
unifiedStorageBigObjectsSupport,experimental,@grafana/search-and-storage,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
155 rolePickerDrawer experimental @grafana/identity-access-team false false false
156 unifiedStorageSearch experimental @grafana/search-and-storage false false false
157 unifiedStorageSearchSprinkles experimental @grafana/search-and-storage false false false
158 unifiedStorageUseFullNgram experimental @grafana/search-and-storage false false false
159 managedDualWriter experimental @grafana/search-and-storage false false false
160 pluginsSriChecks GA @grafana/plugins-platform-backend false false false
161 unifiedStorageBigObjectsSupport experimental @grafana/search-and-storage false false false
pkg/services/featuremgmt/toggles_gen.go
+4
View File
@@ -631,6 +631,10 @@ const (
// Enable sprinkles on unified storage search
FlagUnifiedStorageSearchSprinkles = "unifiedStorageSearchSprinkles"
// FlagUnifiedStorageUseFullNgram
// Use full n-gram indexing instead of edge n-gram for unified storage search
FlagUnifiedStorageUseFullNgram = "unifiedStorageUseFullNgram"
// FlagManagedDualWriter
// Pick the dual write mode from database configs
FlagManagedDualWriter = "managedDualWriter"
pkg/storage/unified/search/bleve.go
+7 -1
View File
@@ -77,6 +77,8 @@ type BleveOptions struct {
MinBuildVersion *semver.Version // Minimum build version for reusing file-based indexes. Ignored if nil.
Logger *slog.Logger
UseFullNgram bool
}
type bleveBackend struct {
@@ -89,6 +91,9 @@ type bleveBackend struct {
indexMetrics *resource.BleveIndexMetrics
// if true will use ngram instead of edge_ngram for title indexes. See custom_analyzers.go
useFullNgram bool
metricsUpdaterCancel func()
metricsUpdaterWg sync.WaitGroup
}
@@ -130,6 +135,7 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resou
cache: map[resource.NamespacedResource]*bleveIndex{},
opts: opts,
indexMetrics: indexMetrics,
useFullNgram: opts.UseFullNgram,
}
if be.indexMetrics != nil {
@@ -314,7 +320,7 @@ func (b *bleveBackend) BuildIndex(
attribute.String("reason", indexBuildReason),
)
mapper, err := GetBleveMappings(fields)
mapper, err := GetBleveMappings(fields, b.useFullNgram)
if err != nil {
return nil, err
}
pkg/storage/unified/search/bleve_integration_test.go
+48 -1
View File
@@ -20,6 +20,29 @@ func TestBleveSearchBackend(t *testing.T) {
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
FileThreshold: 5,
UseFullNgram: false,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
require.NotNil(t, backend)
t.Cleanup(backend.closeAllIndexes)
return backend
}, &unitest.TestOptions{
NSPrefix: "bleve-test",
})
}
func TestBleveSearchBackendFullNgramEnabled(t *testing.T) {
// Run the search backend test suite
unitest.RunSearchBackendTest(t, func(ctx context.Context) resource.SearchBackend {
tempDir := t.TempDir()
// Create a new bleve backend
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
FileThreshold: 5,
UseFullNgram: true,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
require.NotNil(t, backend)
@@ -44,7 +67,31 @@ func TestSearchBackendBenchmark(t *testing.T) {
// Create a new bleve backend
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
Root: tempDir,
UseFullNgram: false,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
require.NotNil(t, backend)
t.Cleanup(backend.closeAllIndexes)
unitest.BenchmarkSearchBackend(t, backend, opts)
}
func TestSearchBackendBenchmarkFullNgramEnabled(t *testing.T) {
opts := &unitest.BenchmarkOptions{
NumResources: 10000,
Concurrency: 1, // For now we only want to test the write throughput
NumNamespaces: 1,
NumGroups: 1,
NumResourceTypes: 1,
}
tempDir := t.TempDir()
// Create a new bleve backend
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
UseFullNgram: true,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
require.NotNil(t, backend)
pkg/storage/unified/search/bleve_mappings.go
+2 -2
View File
@@ -9,10 +9,10 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
func GetBleveMappings(fields resource.SearchableDocumentFields, useFullNgram bool) (mapping.IndexMapping, error) {
mapper := bleve.NewIndexMapping()
err := RegisterCustomAnalyzers(mapper)
err := RegisterCustomAnalyzers(mapper, useFullNgram)
if err != nil {
return nil, err
}
pkg/storage/unified/search/bleve_mappings_test.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
func TestDocumentMapping(t *testing.T) {
mappings, err := search.GetBleveMappings(nil)
mappings, err := search.GetBleveMappings(nil, false)
require.NoError(t, err)
data := resource.IndexableDocument{
Title: "title",
pkg/storage/unified/search/bleve_performance_test.go
+12 -4
View File
@@ -13,14 +13,14 @@ import (
"github.com/stretchr/testify/require"
)
func setupIndex(b testing.TB) resource.ResourceIndex {
func setupIndex(b testing.TB, useFullNgram bool) resource.ResourceIndex {
// size := 1000000 // TODO: 200k documents standard size?
size := 200000
// batchSize := 1000 slower 8s (for 200k documents) - 34s (for 1M documents)
// batchSize := 10000 // faster 5s (for 200k documents) - 27s (for 1M documents)
batchSize := 100000 // fasterer 3.5s (for 200k documents) - 27s (for 1M documents)
writer := newTestWriter(size, batchSize)
return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer)
return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer, useFullNgram)
}
const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms)
@@ -32,12 +32,20 @@ const verbose = false
// changes the the indexer settings can cause unforeseen performance issues ( for example: using wildcard queries )
// this will fail if the stats exceed the "normal" thresholds
func BenchmarkBleveQuery(b *testing.B) {
testIndex := setupIndex(b, false)
runBenchmark(b, testIndex)
}
func BenchmarkBleveQueryFullNgram(b *testing.B) {
testIndex := setupIndex(b, true)
runBenchmark(b, testIndex)
}
func runBenchmark(b *testing.B, testIndex resource.ResourceIndex) {
var memStatsStart runtime.MemStats
var memStatsAfterIndex runtime.MemStats
runtime.ReadMemStats(&memStatsStart)
testIndex := setupIndex(b)
runtime.ReadMemStats(&memStatsAfterIndex)
allocDiff := memStatsAfterIndex.Alloc - memStatsStart.Alloc
pkg/storage/unified/search/bleve_search_test.go
+178 -143
View File
@@ -58,155 +58,189 @@ func TestCanSearchByTitle(t *testing.T) {
Resource: "dashboards",
}
t.Run("when query is empty, sort documents by title instead of search score", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "bbb",
"name2": "aaa",
})
for _, useFullNgram := range []bool{false, true} {
name := "when useFullNgram feature flag is "
if useFullNgram {
name += "enabled"
} else {
name += "disabled"
}
checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"})
})
t.Run(name, func(t *testing.T) {
t.Run("and query is empty, sort documents by title instead of search score", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "bbb",
"name2": "aaa",
})
t.Run("will boost phrase match query over match query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a hello",
"name2": "we want hello",
})
checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"})
})
t.Run("will prioritize matches", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Asserts Dashboards",
"name2": "New dashboard 10",
})
checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"})
})
t.Run("all terms must match", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Dashboard",
"name2": "New dashboard 10",
})
checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"})
})
t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "we want hello pls",
"name2": "we want hello",
})
checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"})
})
t.Run("title with numbers will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "A123456",
})
// search for prefix of title with mix of chars and numbers
checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"})
// search for whole title
checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"})
// case insensive search for partial title
checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"})
})
t.Run("title will match escaped characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "what\"s up",
"name2": "what\"s that",
})
checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"})
checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"})
})
t.Run("title search will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
})
// search by entire phrase
checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"})
// search for word at start
checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"})
// search for word larger than ngram max size
checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"})
// search for word at end
checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"})
// can search for word substring anchored at start of word (edge ngram)
checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"})
// can search for multiple, non-consecutive words in title
checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"})
// can search for multiple, non-consecutive words in title
checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"})
})
t.Run("title search will NOT match documents", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
"name2": "A0456",
"name3": "mash-A02382-10",
})
// word that doesn't exist
checkSearchQuery(t, index, newTestQuery("cats"), nil)
// string shorter than 3 chars (ngam min)
checkSearchQuery(t, index, newTestQuery("ma"), nil)
// substring that doesn't exist
checkSearchQuery(t, index, newTestQuery("A01"), nil)
})
t.Run("title search with character will match one document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "foo",
})
for i, v := range search.TermCharacters {
name := fmt.Sprintf("name%d", i)
title := fmt.Sprintf(`test foo%d%sbar`, i, v)
indexDocumentsWithTitles(t, index, key, map[string]string{
name: title,
checkSearchQuery(t, index, newTestQuery(""), []string{"name2", "name1"})
})
checkSearchQuery(t, index, newQueryByTitle(title), []string{name})
t.Run("will boost phrase match query over match query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a hello",
"name2": "we want hello",
})
// can search for a title with a term character suffix
checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name})
}
})
checkSearchQuery(t, index, newTestQuery("want hello"), []string{"name2", "name1"})
})
t.Run("title search will ignore terms < 3 characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "new dashboard",
"name2": "new dash",
"name3": "new",
t.Run("will prioritize matches", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Asserts Dashboards",
"name2": "New dashboard 10",
})
checkSearchQuery(t, index, newTestQuery("dashboard"), []string{"name2", "name1"})
})
t.Run("all terms must match", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Dashboard",
"name2": "New dashboard 10",
})
checkSearchQuery(t, index, newTestQuery("dashboard new"), []string{"name2"})
})
t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "we want hello pls",
"name2": "we want hello",
})
checkSearchQuery(t, index, newTestQuery("we want hello"), []string{"name2", "name1"})
})
t.Run("title with numbers will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "A123456",
})
// search for prefix of title with mix of chars and numbers
checkSearchQuery(t, index, newQueryByTitle("A12"), []string{"name1"})
// search for whole title
checkSearchQuery(t, index, newQueryByTitle("A123456"), []string{"name1"})
// case insensive search for partial title
checkSearchQuery(t, index, newQueryByTitle("a1234"), []string{"name1"})
})
t.Run("title will match escaped characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "what\"s up",
"name2": "what\"s that",
})
checkSearchQuery(t, index, newQueryByTitle("what\"s up"), []string{"name1"})
checkSearchQuery(t, index, newQueryByTitle("what\"s"), []string{"name2", "name1"})
})
t.Run("title search will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
})
// search by entire phrase
checkSearchQuery(t, index, newTestQuery("I want to say a wonderfully Hello to the WORLD! Hello-world"), []string{"name1"})
// search for word at start
checkSearchQuery(t, index, newTestQuery("hello"), []string{"name1"})
// search for word larger than ngram max size
checkSearchQuery(t, index, newTestQuery("wonderfully"), []string{"name1"})
// search for word at end
checkSearchQuery(t, index, newTestQuery("world"), []string{"name1"})
// can search for word substring anchored at start of word (edge ngram)
checkSearchQuery(t, index, newTestQuery("worl"), []string{"name1"})
// can search for multiple, non-consecutive words in title
checkSearchQuery(t, index, newTestQuery("hello world"), []string{"name1"})
// can search for multiple, non-consecutive words in title
checkSearchQuery(t, index, newTestQuery("hello-world"), []string{"name1"})
})
t.Run("title search will NOT match documents", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
"name2": "A0456",
"name3": "mash-A02382-10",
})
// word that doesn't exist
checkSearchQuery(t, index, newTestQuery("cats"), nil)
// string shorter than 3 chars (ngram min)
checkSearchQuery(t, index, newTestQuery("ma"), nil)
// substring that doesn't exist
checkSearchQuery(t, index, newTestQuery("A01"), nil)
})
t.Run("title search with character will match one document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "foo",
})
for i, v := range search.TermCharacters {
name := fmt.Sprintf("name%d", i)
title := fmt.Sprintf(`test foo%d%sbar`, i, v)
indexDocumentsWithTitles(t, index, key, map[string]string{
name: title,
})
checkSearchQuery(t, index, newQueryByTitle(title), []string{name})
// can search for a title with a term character suffix
checkSearchQuery(t, index, newQueryByTitle(fmt.Sprintf(`foo%d%s`, i, v)), []string{name})
}
})
t.Run("title search will ignore terms < 3 characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "new dashboard",
"name2": "new dash",
"name3": "new",
})
// matches everything
checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"})
// ignore terms shorter than 3 chars
checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"})
// include terms shorter that are exactly 3 chars
checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"})
})
name := "title search will%smatch term in the middle/end"
if useFullNgram {
name = fmt.Sprintf(name, " ")
} else {
name = fmt.Sprintf(name, " NOT ")
}
t.Run(name, func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "new dashboard",
"name2": "new dash",
"name3": "somedash",
})
if useFullNgram {
checkSearchQuery(t, index, newTestQuery("ash"), []string{"name2", "name3", "name1"})
checkSearchQuery(t, index, newTestQuery("ome"), []string{"name3"})
} else {
checkSearchQuery(t, index, newTestQuery("ash"), nil)
checkSearchQuery(t, index, newTestQuery("ome"), nil)
}
})
})
// matches everything
checkSearchQuery(t, index, newTestQuery("new"), []string{"name3", "name2", "name1"})
// ignore terms shorter than 3 chars
checkSearchQuery(t, index, newTestQuery("new d"), []string{"name3", "name2", "name1"})
// include terms shorter that are exactly 3 chars
checkSearchQuery(t, index, newTestQuery("new das"), []string{"name2", "name1"})
})
}
}
func newTestQuery(query string) *resourcepb.ResourceSearchRequest {
@@ -237,7 +271,7 @@ func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest {
}
}
func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn) resource.ResourceIndex {
func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex {
key := &resourcepb.ResourceKey{
Namespace: "default",
Group: "dashboard.grafana.app",
@@ -247,6 +281,7 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize
Root: t.TempDir(),
FileThreshold: threshold, // use in-memory for tests
BatchSize: int(batchSize),
UseFullNgram: useFullNgram,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
pkg/storage/unified/search/bleve_test.go
+30 -8
View File
@@ -46,6 +46,34 @@ func TestMain(m *testing.M) {
}
func TestBleveBackend(t *testing.T) {
tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
require.NoError(t, err)
backend, err := NewBleveBackend(BleveOptions{
Root: tmpdir,
FileThreshold: 5, // with more than 5 items we create a file on disk
UseFullNgram: false,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
testBleveBackend(t, backend)
}
func TestBleveBackendFullNgramEnabled(t *testing.T) {
tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
require.NoError(t, err)
backend, err := NewBleveBackend(BleveOptions{
Root: tmpdir,
FileThreshold: 5, // with more than 5 items we create a file on disk
UseFullNgram: true,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
testBleveBackend(t, backend)
}
func testBleveBackend(t *testing.T, backend *bleveBackend) {
dashboardskey := &resourcepb.ResourceKey{
Namespace: "default",
Group: "dashboard.grafana.app",
@@ -56,14 +84,6 @@ func TestBleveBackend(t *testing.T) {
Group: "folder.grafana.app",
Resource: "folders",
}
tmpdir, err := os.MkdirTemp("", "grafana-bleve-test")
require.NoError(t, err)
backend, err := NewBleveBackend(BleveOptions{
Root: tmpdir,
FileThreshold: 5, // with more than 5 items we create a file on disk
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
t.Cleanup(backend.Stop)
@@ -771,6 +791,7 @@ func setupBleveBackend(t *testing.T, options ...setupOption) (*bleveBackend, pro
IndexCacheTTL: defaultIndexCacheTTL,
Logger: slog.New(logtest.NewNopHandler(t)),
BuildVersion: buildVersion,
UseFullNgram: false,
}
for _, opt := range options {
opt(&opts)
@@ -1467,6 +1488,7 @@ func TestInvalidBuildVersion(t *testing.T) {
opts := BleveOptions{
Root: t.TempDir(),
BuildVersion: "invalid",
UseFullNgram: false,
}
_, err := NewBleveBackend(opts, tracing.NewNoopTracerService(), nil)
require.ErrorContains(t, err, "cannot parse build version")
pkg/storage/unified/search/custom_analyzers.go
+21 -10
View File
@@ -4,6 +4,7 @@ import (
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/token/edgengram"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/ngram"
"github.com/blevesearch/bleve/v2/analysis/token/unique"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace"
"github.com/blevesearch/bleve/v2/mapping"
@@ -11,23 +12,33 @@ import (
const TITLE_ANALYZER = "title_analyzer"
const EDGE_NGRAM_MIN_TOKEN = 3.0
const tokenFilterName = "ngram_filter"
func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl) error {
return registerTitleAnalyzer(mapper)
func RegisterCustomAnalyzers(mapper *mapping.IndexMappingImpl, useFullNgram bool) error {
return registerTitleAnalyzer(mapper, useFullNgram)
}
// The registerTitleAnalyzer function defines a custom analyzer for the title field.
// The edgeNgramTokenFilter will create n-grams anchored to the front of each token.
// For example, the token "hello" will be tokenized into "hel", "hell", "hello".
func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error {
// Define an N-Gram tokenizer (for substring search)
edgeNgramTokenFilter := map[string]interface{}{
// The registerTitleAnalyzer function defines a custom analyzer using edge n-gram or full n-gram
func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl, useFullNgram bool) error {
// The edgengram tokenFilter will create grams anchored to the front of each token.
// For example, the token "hello" will be tokenized into "hel", "hell", "hello".
tokenFilter := map[string]interface{}{
"type": edgengram.Name,
"min": EDGE_NGRAM_MIN_TOKEN,
"max": 10.0,
"back": edgengram.FRONT,
}
err := mapper.AddCustomTokenFilter("edge_ngram_filter", edgeNgramTokenFilter)
if useFullNgram {
// The ngram tokenFilter will create additional grams in the middle of each token.
// For example, the token "hello" will be tokenized into "hel", "hell", "hello", "ell", "ello", "llo".
tokenFilter = map[string]interface{}{
"type": ngram.Name,
"min": EDGE_NGRAM_MIN_TOKEN,
"max": 10.0,
}
}
err := mapper.AddCustomTokenFilter(tokenFilterName, tokenFilter)
if err != nil {
return err
}
@@ -36,7 +47,7 @@ func registerTitleAnalyzer(mapper *mapping.IndexMappingImpl) error {
ngramAnalyzer := map[string]interface{}{
"type": custom.Name,
"tokenizer": whitespace.Name,
"token_filters": []string{"edge_ngram_filter", lowercase.Name, unique.Name},
"token_filters": []string{tokenFilterName, lowercase.Name, unique.Name},
//"char_filters": //TODO IF NEEDED
}
pkg/storage/unified/search/options.go
+1
View File
@@ -43,6 +43,7 @@ func NewSearchOptions(features featuremgmt.FeatureToggles, cfg *setting.Cfg, tra
BuildVersion: cfg.BuildVersion,
MaxFileIndexAge: cfg.MaxFileIndexAge,
MinBuildVersion: minVersion,
UseFullNgram: features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageUseFullNgram),
}, tracer, indexMetrics)
if err != nil {
pkg/storage/unified/sql/test/integration_test.go
-2
View File
@@ -110,8 +110,6 @@ func TestIntegrationSearchAndStorage(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, search)
t.Cleanup(search.Stop)
// Create a new resource backend
storage := newTestBackend(t, false, 0)
require.NotNil(t, storage)