Rebuild search indexes asynchronously (#111829)

* Add "debouncer" queue, which can combine incoming elements.

* Rebuild indexes asynchronously.

* Remove duplicate method.

* Fix bleve tests.

* Extracted combineRebuildRequests and added test for it.

* Add TestShouldRebuildIndex

* Added TestFindIndexesForRebuild

* Added TestFindIndexesForRebuild

* Introduce index_rebuild_workers option.

* Add metric for rebuild queue length.

* Add TestRebuildIndexes.

* Fix import.

* Linter, review feedback.

pkg/setting/setting.go

@@ -575,13 +575,14 @@ type Cfg struct {
 	MaxPageSizeBytes                           int
 	IndexPath                                  string
 	IndexWorkers                               int
+	IndexRebuildWorkers                        int
 	IndexMaxBatchSize                          int
 	IndexFileThreshold                         int
 	IndexMinCount                              int
 	IndexRebuildInterval                       time.Duration
 	IndexCacheTTL                              time.Duration
-	MaxFileIndexAge                            time.Duration // Max age of file-based indexes. Index older than this will not be reused between restarts.
-	MinFileIndexBuildVersion                   string        // Minimum version of Grafana that built the file-based index. If index was built with older Grafana, it will not be reused between restarts.
+	MaxFileIndexAge                            time.Duration // Max age of file-based indexes. Index older than this will be rebuilt asynchronously.
+	MinFileIndexBuildVersion                   string        // Minimum version of Grafana that built the file-based index. If index was built with older Grafana, it will be rebuilt asynchronously.
 	EnableSharding                             bool
 	QOSEnabled                                 bool
 	QOSNumberWorker                            int

pkg/setting/setting_unified_storage.go

@@ -54,6 +54,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 	cfg.MaxPageSizeBytes = section.Key("max_page_size_bytes").MustInt(0)
 	cfg.IndexPath = section.Key("index_path").String()
 	cfg.IndexWorkers = section.Key("index_workers").MustInt(10)
+	cfg.IndexRebuildWorkers = section.Key("index_rebuild_workers").MustInt(5)
 	cfg.IndexMaxBatchSize = section.Key("index_max_batch_size").MustInt(100)
 	cfg.EnableSharding = section.Key("enable_sharding").MustBool(false)
 	cfg.QOSEnabled = section.Key("qos_enabled").MustBool(false)