Unified Storage/Search: Add max count config for indexing (#107255)

* Add max count config for indexing
* Build empty index when max count is exceeded
* Address linting
* Refactor buildIndexes
* Add test for max count threshold
* Update test doc comments
* Refactor TestBuildIndexes_MaxCountThreshold to not use mock framework
* Rename mocks used in TestBuildIndexes_MaxCountThreshold

* Refactor mockResourceIndex

* Test setting of indexing threshold configs

* Tweak comments, log

* Fix logging in buildEmptyIndex

* Export and reuse TestDocumentBuilderSupplier

* Reuse MockResourceIndex

pkg/setting/setting.go

@@ -563,6 +563,7 @@ type Cfg struct {
 	IndexMaxBatchSize                          int
 	IndexFileThreshold                         int
 	IndexMinCount                              int
+	IndexMaxCount                              int
 	IndexRebuildInterval                       time.Duration
 	IndexCacheTTL                              time.Duration
 	EnableSharding                             bool

pkg/setting/setting_unified_storage.go

@@ -64,6 +64,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 	cfg.InstanceID = section.Key("instance_id").String()
 	cfg.IndexFileThreshold = section.Key("index_file_threshold").MustInt(10)
 	cfg.IndexMinCount = section.Key("index_min_count").MustInt(1)
+	cfg.IndexMaxCount = section.Key("index_max_count").MustInt(0)
 	// default to 24 hours because usage insights summarizes the data every 24 hours
 	cfg.IndexRebuildInterval = section.Key("index_rebuild_interval").MustDuration(24 * time.Hour)
 	cfg.IndexCacheTTL = section.Key("index_cache_ttl").MustDuration(10 * time.Minute)

pkg/setting/setting_unified_storage_test.go

@@ -28,6 +28,16 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 		_, err = s.NewKey("dataSyncerInterval", "10m")
 		assert.NoError(t, err)
 
+		// Add unified_storage section for index settings
+		unifiedStorageSection, err := cfg.Raw.NewSection("unified_storage")
+		assert.NoError(t, err)
+
+		_, err = unifiedStorageSection.NewKey("index_min_count", "5")
+		assert.NoError(t, err)
+
+		_, err = unifiedStorageSection.NewKey("index_max_count", "1000")
+		assert.NoError(t, err)
+
 		cfg.setUnifiedStorageConfig()
 
 		value, exists := cfg.UnifiedStorage["playlists.playlist.grafana.app"]
@@ -39,5 +49,22 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 			DataSyncerRecordsLimit:               1001,
 			DataSyncerInterval:                   time.Minute * 10,
 		})
+
+		// Test that index settings are correctly parsed
+		assert.Equal(t, 5, cfg.IndexMinCount)
+		assert.Equal(t, 1000, cfg.IndexMaxCount)
+	})
+
+	t.Run("read unified_storage configs with defaults", func(t *testing.T) {
+		cfg := NewCfg()
+		err := cfg.Load(CommandLineArgs{HomePath: "../../", Config: "../../conf/defaults.ini"})
+		assert.NoError(t, err)
+
+		// Don't add any custom index settings, test defaults
+		cfg.setUnifiedStorageConfig()
+
+		// Test that default index settings are applied
+		assert.Equal(t, 1, cfg.IndexMinCount)
+		assert.Equal(t, 0, cfg.IndexMaxCount)
 	})
 }