fix(unified-storage): remove bleve index metric from global scope (#101825)

* refactor grafana_index_server_index_size to calculate in a goroutine instead of at scrape time and remove grafana_index_server_indexed_docs metric

* use wire to inject bleve index metrics

* remove sprinkles metrics from bleve index metrics

* log error when trying to calculate file index size and bump interval to 1m instead of 5s

pkg/storage/unified/sql/server.go

@@ -18,7 +18,7 @@ import (
 
 // Creates a new ResourceServer
 func NewResourceServer(db infraDB.DB, cfg *setting.Cfg,
-	tracer tracing.Tracer, reg prometheus.Registerer, ac types.AccessClient, searchOptions resource.SearchOptions, storageMetrics *resource.StorageMetrics) (resource.ResourceServer, error) {
+	tracer tracing.Tracer, reg prometheus.Registerer, ac types.AccessClient, searchOptions resource.SearchOptions, storageMetrics *resource.StorageMetrics, indexMetrics *resource.BleveIndexMetrics) (resource.ResourceServer, error) {
 	apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver")
 	opts := resource.ResourceServerOptions{
 		Tracer: tracer,
@@ -55,6 +55,7 @@ func NewResourceServer(db infraDB.DB, cfg *setting.Cfg,
 	opts.Diagnostics = store
 	opts.Lifecycle = store
 	opts.Search = searchOptions
+	opts.IndexMetrics = indexMetrics
 
 	rs, err := resource.NewResourceServer(opts)
 	if err != nil {

pkg/storage/unified/sql/service.go

@@ -52,6 +52,7 @@ type service struct {
 	log            log.Logger
 	reg            prometheus.Registerer
 	storageMetrics *resource.StorageMetrics
+	indexMetrics   *resource.BleveIndexMetrics
 
 	docBuilders resource.DocumentBuilderSupplier
 }
@@ -64,6 +65,7 @@ func ProvideUnifiedStorageGrpcService(
 	reg prometheus.Registerer,
 	docBuilders resource.DocumentBuilderSupplier,
 	storageMetrics *resource.StorageMetrics,
+	indexMetrics *resource.BleveIndexMetrics,
 ) (UnifiedStorageGrpcService, error) {
 	tracingCfg, err := tracing.ProvideTracingConfig(cfg)
 	if err != nil {
@@ -96,6 +98,7 @@ func ProvideUnifiedStorageGrpcService(
 		reg:            reg,
 		docBuilders:    docBuilders,
 		storageMetrics: storageMetrics,
+		indexMetrics:   indexMetrics,
 	}
 
 	// This will be used when running as a dskit service
@@ -110,12 +113,12 @@ func (s *service) start(ctx context.Context) error {
 		return err
 	}
 
-	searchOptions, err := search.NewSearchOptions(s.features, s.cfg, s.tracing, s.docBuilders, s.reg)
+	searchOptions, err := search.NewSearchOptions(s.features, s.cfg, s.tracing, s.docBuilders, s.indexMetrics)
 	if err != nil {
 		return err
 	}
 
-	server, err := NewResourceServer(s.db, s.cfg, s.tracing, s.reg, authzClient, searchOptions, s.storageMetrics)
+	server, err := NewResourceServer(s.db, s.cfg, s.tracing, s.reg, authzClient, searchOptions, s.storageMetrics, s.indexMetrics)
 	if err != nil {
 		return err
 	}

pkg/storage/unified/sql/test/integration_test.go

@@ -83,7 +83,7 @@ func TestClientServer(t *testing.T) {
 
 	features := featuremgmt.WithFeatures()
 
-	svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil, prometheus.NewPedanticRegistry(), nil, nil)
+	svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil, prometheus.NewPedanticRegistry(), nil, nil, nil)
 	require.NoError(t, err)
 	var client resource.ResourceStoreClient