Unified Storage/Search: Add max count config for indexing (#107255)

* Add max count config for indexing
* Build empty index when max count is exceeded
* Address linting
* Refactor buildIndexes
* Add test for max count threshold
* Update test doc comments
* Refactor TestBuildIndexes_MaxCountThreshold to not use mock framework
* Rename mocks used in TestBuildIndexes_MaxCountThreshold

* Refactor mockResourceIndex

* Test setting of indexing threshold configs

* Tweak comments, log

* Fix logging in buildEmptyIndex

* Export and reuse TestDocumentBuilderSupplier

* Reuse MockResourceIndex
This commit is contained in:
Arati R.
2025-06-27 14:00:39 +02:00
committed by GitHub
parent 3020794b60
commit 0982cfd9a0
10 changed files with 356 additions and 76 deletions
@@ -28,6 +28,16 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
_, err = s.NewKey("dataSyncerInterval", "10m")
assert.NoError(t, err)
// Add unified_storage section for index settings
unifiedStorageSection, err := cfg.Raw.NewSection("unified_storage")
assert.NoError(t, err)
_, err = unifiedStorageSection.NewKey("index_min_count", "5")
assert.NoError(t, err)
_, err = unifiedStorageSection.NewKey("index_max_count", "1000")
assert.NoError(t, err)
cfg.setUnifiedStorageConfig()
value, exists := cfg.UnifiedStorage["playlists.playlist.grafana.app"]
@@ -39,5 +49,22 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
DataSyncerRecordsLimit: 1001,
DataSyncerInterval: time.Minute * 10,
})
// Test that index settings are correctly parsed
assert.Equal(t, 5, cfg.IndexMinCount)
assert.Equal(t, 1000, cfg.IndexMaxCount)
})
t.Run("read unified_storage configs with defaults", func(t *testing.T) {
cfg := NewCfg()
err := cfg.Load(CommandLineArgs{HomePath: "../../", Config: "../../conf/defaults.ini"})
assert.NoError(t, err)
// Don't add any custom index settings, test defaults
cfg.setUnifiedStorageConfig()
// Test that default index settings are applied
assert.Equal(t, 1, cfg.IndexMinCount)
assert.Equal(t, 0, cfg.IndexMaxCount)
})
}