public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unused index_max_batch_size option to bleve backend. (#112171)

Browse Source 
* Remove unused index_max_batch_size option to bleve backend.

* Remove batchSize argument.
This commit is contained in:
Peter Štibraný
2025-10-08 17:04:38 +02:00
committed by GitHub
parent 3f40947cd6
commit 3c6768460c
6 changed files with 14 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/setting/setting.go
-1
View File
@@ -577,7 +577,6 @@ type Cfg struct {
IndexPath string
IndexWorkers int
IndexRebuildWorkers int
IndexMaxBatchSize int
IndexFileThreshold int
IndexMinCount int
IndexRebuildInterval time.Duration
pkg/setting/setting_unified_storage.go
-1
View File
@@ -55,7 +55,6 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
cfg.IndexPath = section.Key("index_path").String()
cfg.IndexWorkers = section.Key("index_workers").MustInt(10)
cfg.IndexRebuildWorkers = section.Key("index_rebuild_workers").MustInt(5)
cfg.IndexMaxBatchSize = section.Key("index_max_batch_size").MustInt(100)
cfg.EnableSharding = section.Key("enable_sharding").MustBool(false)
cfg.QOSEnabled = section.Key("qos_enabled").MustBool(false)
cfg.QOSNumberWorker = section.Key("qos_num_worker").MustInt(16)
pkg/storage/unified/search/bleve.go
-4
View File
@@ -65,10 +65,6 @@ type BleveOptions struct {
// The resource count where values switch from memory to file based
FileThreshold int64
// How big should a batch get before flushing
// ?? not totally sure the units
BatchSize int
// Index cache TTL for bleve indices. 0 disables expiration for in-memory indexes.
// Also used for file-based indexes, if they are not owned by this instance, and they are not fetched from the cache recently.
IndexCacheTTL time.Duration
pkg/storage/unified/search/bleve_performance_test.go
+1 -1
View File
@@ -20,7 +20,7 @@ func setupIndex(b testing.TB, useFullNgram bool) resource.ResourceIndex {
// batchSize := 10000 // faster 5s (for 200k documents) - 27s (for 1M documents)
batchSize := 100000 // fasterer 3.5s (for 200k documents) - 27s (for 1M documents)
writer := newTestWriter(size, batchSize)
return newTestDashboardsIndex(b, 1, int64(size), int64(batchSize), writer, useFullNgram)
return newTestDashboardsIndex(b, 1, int64(size), writer, useFullNgram)
}
const maxAllowedTime = 20 * time.Millisecond // Reasonable (can vary per env) performance threshold per query (e.g., 20ms)
pkg/storage/unified/search/bleve_search_test.go
+13 -14
View File
@@ -68,7 +68,7 @@ func TestCanSearchByTitle(t *testing.T) {
t.Run(name, func(t *testing.T) {
t.Run("and query is empty, sort documents by title instead of search score", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "bbb",
"name2": "aaa",
@@ -78,7 +78,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("will boost phrase match query over match query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a hello",
"name2": "we want hello",
@@ -88,7 +88,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("will prioritize matches", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Asserts Dashboards",
"name2": "New dashboard 10",
@@ -98,7 +98,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("all terms must match", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "Dashboard",
"name2": "New dashboard 10",
@@ -108,7 +108,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("will boost exact match query over match phrase query results", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "we want hello pls",
"name2": "we want hello",
@@ -118,7 +118,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title with numbers will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "A123456",
})
@@ -132,7 +132,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title will match escaped characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "what\"s up",
"name2": "what\"s that",
@@ -143,7 +143,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title search will match document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
})
@@ -166,7 +166,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title search will NOT match documents", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "I want to say a wonderfully Hello to the WORLD! Hello-world",
"name2": "A0456",
@@ -182,7 +182,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title search with character will match one document", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "foo",
})
@@ -202,7 +202,7 @@ func TestCanSearchByTitle(t *testing.T) {
})
t.Run("title search will ignore terms < 3 characters", func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "new dashboard",
"name2": "new dash",
@@ -224,7 +224,7 @@ func TestCanSearchByTitle(t *testing.T) {
name = fmt.Sprintf(name, " NOT ")
}
t.Run(name, func(t *testing.T) {
index := newTestDashboardsIndex(t, threshold, 2, 2, noop, useFullNgram)
index := newTestDashboardsIndex(t, threshold, 2, noop, useFullNgram)
indexDocumentsWithTitles(t, index, key, map[string]string{
"name1": "new dashboard",
"name2": "new dash",
@@ -271,7 +271,7 @@ func newQueryByTitle(query string) *resourcepb.ResourceSearchRequest {
}
}
func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex {
func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer resource.BuildFn, useFullNgram bool) resource.ResourceIndex {
key := &resourcepb.ResourceKey{
Namespace: "default",
Group: "dashboard.grafana.app",
@@ -280,7 +280,6 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize
backend, err := search.NewBleveBackend(search.BleveOptions{
Root: t.TempDir(),
FileThreshold: threshold, // use in-memory for tests
BatchSize: int(batchSize),
UseFullNgram: useFullNgram,
}, tracing.NewNoopTracerService(), nil)
require.NoError(t, err)
pkg/storage/unified/search/options.go
-1
View File
@@ -43,7 +43,6 @@ func NewSearchOptions(
bleve, err := NewBleveBackend(BleveOptions{
Root: root,
FileThreshold: int64(cfg.IndexFileThreshold), // fewer than X items will use a memory index
BatchSize: cfg.IndexMaxBatchSize, // This is the batch size for how many objects to add to the index at once
IndexCacheTTL: cfg.IndexCacheTTL, // How long to keep the index cache in memory
BuildVersion: cfg.BuildVersion,
UseFullNgram: features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageUseFullNgram),