diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 89a42e9d56d..49d9dc9511b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -577,7 +577,6 @@ type Cfg struct { IndexPath string IndexWorkers int IndexRebuildWorkers int - IndexMaxBatchSize int IndexFileThreshold int IndexMinCount int IndexRebuildInterval time.Duration diff --git a/pkg/setting/setting_unified_storage.go b/pkg/setting/setting_unified_storage.go index 914d5f789d7..759d076e51c 100644 --- a/pkg/setting/setting_unified_storage.go +++ b/pkg/setting/setting_unified_storage.go @@ -55,7 +55,6 @@ func (cfg *Cfg) setUnifiedStorageConfig() { cfg.IndexPath = section.Key("index_path").String() cfg.IndexWorkers = section.Key("index_workers").MustInt(10) cfg.IndexRebuildWorkers = section.Key("index_rebuild_workers").MustInt(5) - cfg.IndexMaxBatchSize = section.Key("index_max_batch_size").MustInt(100) cfg.EnableSharding = section.Key("enable_sharding").MustBool(false) cfg.QOSEnabled = section.Key("qos_enabled").MustBool(false) cfg.QOSNumberWorker = section.Key("qos_num_worker").MustInt(16) diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 190322bab7c..0f3ab8b2614 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -65,10 +65,6 @@ type BleveOptions struct { // The resource count where values switch from memory to file based FileThreshold int64 - // How big should a batch get before flushing - // ?? not totally sure the units - BatchSize int - // Index cache TTL for bleve indices. 0 disables expiration for in-memory indexes. // Also used for file-based indexes, if they are not owned by this instance, and they are not fetched from the cache recently. IndexCacheTTL time.Duration diff --git a/pkg/storage/unified/search/bleve_performance_test.go b/pkg/storage/unified/search/bleve_performance_test.go index f8f869180cf..643e6a70f14 100644 --- a/pkg/storage/unified/search/bleve_performance_test.go +++ b/pkg/storage/unified/search/bleve_performance_test.go @@ -20,7 +20,7 @@ func setupIndex(b testing.TB, useFullNgram bool) resource.ResourceIndex { // batchSize := 10000 // faster 5s (for 200k documents) - 27s (for 1M documents) batchSize := 100000 // fasterer 3.5s (for 200k documents) - 27s (for 1M documents) writer := newTestWriter(size, batchSize) - return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer, useFullNgram) + return newTestDashboardsIndex(b, 1, int64(size), writer, useFullNgram) } const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms) diff --git a/pkg/storage/unified/search/bleve_search_test.go b/pkg/storage/unified/search/bleve_search_test.go index 860df9f268a..476f35985a1 100644 --- a/pkg/storage/unified/search/bleve_search_test.go +++ b/pkg/storage/unified/search/bleve_search_test.go @@ -68,7 +68,7 @@ func TestCanSearchByTitle(t *testing.T) { t.Run(name, func(t *testing.T) { t.Run("and query is empty, sort documents by title instead of search score", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "bbb", "name2": "aaa", @@ -78,7 +78,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will boost phrase match query over match query results", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a hello", "name2": "we want hello", @@ -88,7 +88,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will prioritize matches", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "Asserts Dashboards", "name2": "New dashboard 10", @@ -98,7 +98,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("all terms must match", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "Dashboard", "name2": "New dashboard 10", @@ -108,7 +108,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("will boost exact match query over match phrase query results", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "we want hello pls", "name2": "we want hello", @@ -118,7 +118,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title with numbers will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "A123456", }) @@ -132,7 +132,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title will match escaped characters", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "what\"s up", "name2": "what\"s that", @@ -143,7 +143,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title search will match document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", }) @@ -166,7 +166,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title search will NOT match documents", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "I want to say a wonderfully Hello to the WORLD! Hello-world", "name2": "A0456", @@ -182,7 +182,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title search with character will match one document", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "foo", }) @@ -202,7 +202,7 @@ func TestCanSearchByTitle(t *testing.T) { }) t.Run("title search will ignore terms < 3 characters", func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "new dashboard", "name2": "new dash", @@ -224,7 +224,7 @@ func TestCanSearchByTitle(t *testing.T) { name = fmt.Sprintf(name, " NOT ") } t.Run(name, func(t *testing.T) { - index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram) + index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram) indexDocumentsWithTitles(t, index, key, map[string]string{ "name1": "new dashboard", "name2": "new dash", @@ -271,7 +271,7 @@ func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest { } } -func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex { +func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex { key := &resourcepb.ResourceKey{ Namespace: "default", Group: "dashboard.grafana.app", @@ -280,7 +280,6 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize backend, err := search.NewBleveBackend(search.BleveOptions{ Root: t.TempDir(), FileThreshold: threshold, // use in-memory for tests - BatchSize: int(batchSize), UseFullNgram: useFullNgram, }, tracing.NewNoopTracerService(), nil) require.NoError(t, err) diff --git a/pkg/storage/unified/search/options.go b/pkg/storage/unified/search/options.go index 0d856b586c3..47a24659347 100644 --- a/pkg/storage/unified/search/options.go +++ b/pkg/storage/unified/search/options.go @@ -43,7 +43,6 @@ func NewSearchOptions( bleve, err := NewBleveBackend(BleveOptions{ Root: root, FileThreshold: int64(cfg.IndexFileThreshold), // fewer than X items will use a memory index - BatchSize: cfg.IndexMaxBatchSize, // This is the batch size for how many objects to add to the index at once IndexCacheTTL: cfg.IndexCacheTTL, // How long to keep the index cache in memory BuildVersion: cfg.BuildVersion, UseFullNgram: features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageUseFullNgram),