feat(unified-storage): add qos support for the resource server (#105939)

pkg/storage/unified/sql/server.go

@@ -1,6 +1,7 @@
 package sql
 
 import (
+	"context"
 	"os"
 	"strings"
 
@@ -8,6 +9,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/authlib/types"
+	"github.com/grafana/dskit/services"
 
 	infraDB "github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -17,70 +19,85 @@ import (
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
 )
 
+type QOSEnqueueDequeuer interface {
+	services.Service
+	Enqueue(ctx context.Context, tenantID string, runnable func(ctx context.Context)) error
+	Dequeue(ctx context.Context) (func(ctx context.Context), error)
+}
+
+// ServerOptions contains the options for creating a new ResourceServer
+type ServerOptions struct {
+	DB             infraDB.DB
+	Cfg            *setting.Cfg
+	Tracer         trace.Tracer
+	Reg            prometheus.Registerer
+	AccessClient   types.AccessClient
+	SearchOptions  resource.SearchOptions
+	StorageMetrics *resource.StorageMetrics
+	IndexMetrics   *resource.BleveIndexMetrics
+	Features       featuremgmt.FeatureToggles
+	QOSQueue       QOSEnqueueDequeuer
+}
+
 // Creates a new ResourceServer
-func NewResourceServer(db infraDB.DB, cfg *setting.Cfg,
-	tracer trace.Tracer, reg prometheus.Registerer, ac types.AccessClient,
-	searchOptions resource.SearchOptions, storageMetrics *resource.StorageMetrics,
-	indexMetrics *resource.BleveIndexMetrics, features featuremgmt.FeatureToggles) (resource.ResourceServer, error) {
-	apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver")
-	opts := resource.ResourceServerOptions{
-		Tracer: tracer,
+func NewResourceServer(
+	opts ServerOptions,
+) (resource.ResourceServer, error) {
+	apiserverCfg := opts.Cfg.SectionWithEnvOverrides("grafana-apiserver")
+	serverOptions := resource.ResourceServerOptions{
+		Tracer: opts.Tracer,
 		Blob: resource.BlobConfig{
 			URL: apiserverCfg.Key("blob_url").MustString(""),
 		},
-		Reg: reg,
+		Reg: opts.Reg,
 	}
-	if ac != nil {
-		opts.AccessClient = resource.NewAuthzLimitedClient(ac, resource.AuthzOptions{Tracer: tracer, Registry: reg})
+	if opts.AccessClient != nil {
+		serverOptions.AccessClient = resource.NewAuthzLimitedClient(opts.AccessClient, resource.AuthzOptions{Tracer: opts.Tracer, Registry: opts.Reg})
 	}
 	// Support local file blob
-	if strings.HasPrefix(opts.Blob.URL, "./data/") {
-		dir := strings.Replace(opts.Blob.URL, "./data", cfg.DataPath, 1)
+	if strings.HasPrefix(serverOptions.Blob.URL, "./data/") {
+		dir := strings.Replace(serverOptions.Blob.URL, "./data", opts.Cfg.DataPath, 1)
 		err := os.MkdirAll(dir, 0700)
 		if err != nil {
 			return nil, err
 		}
-		opts.Blob.URL = "file:///" + dir
+		serverOptions.Blob.URL = "file:///" + dir
 	}
 
 	// This is mostly for testing, being able to influence when we paginate
 	// based on the page size during tests.
-	unifiedStorageCfg := cfg.SectionWithEnvOverrides("unified_storage")
+	unifiedStorageCfg := opts.Cfg.SectionWithEnvOverrides("unified_storage")
 	maxPageSizeBytes := unifiedStorageCfg.Key("max_page_size_bytes")
-	opts.MaxPageSizeBytes = maxPageSizeBytes.MustInt(0)
+	serverOptions.MaxPageSizeBytes = maxPageSizeBytes.MustInt(0)
 
-	eDB, err := dbimpl.ProvideResourceDB(db, cfg, tracer)
+	eDB, err := dbimpl.ProvideResourceDB(opts.DB, opts.Cfg, opts.Tracer)
 	if err != nil {
 		return nil, err
 	}
 
-	isHA := isHighAvailabilityEnabled(cfg.SectionWithEnvOverrides("database"),
-		cfg.SectionWithEnvOverrides("resource_api"))
-	withPruner := features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageHistoryPruner)
+	isHA := isHighAvailabilityEnabled(opts.Cfg.SectionWithEnvOverrides("database"),
+		opts.Cfg.SectionWithEnvOverrides("resource_api"))
+	withPruner := opts.Features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageHistoryPruner)
 
 	store, err := NewBackend(BackendOptions{
 		DBProvider:     eDB,
-		Tracer:         tracer,
-		Reg:            reg,
+		Tracer:         opts.Tracer,
+		Reg:            opts.Reg,
 		IsHA:           isHA,
 		withPruner:     withPruner,
-		storageMetrics: storageMetrics,
+		storageMetrics: opts.StorageMetrics,
 	})
 	if err != nil {
 		return nil, err
 	}
-	opts.Backend = store
-	opts.Diagnostics = store
-	opts.Lifecycle = store
-	opts.Search = searchOptions
-	opts.IndexMetrics = indexMetrics
+	serverOptions.Backend = store
+	serverOptions.Diagnostics = store
+	serverOptions.Lifecycle = store
+	serverOptions.Search = opts.SearchOptions
+	serverOptions.IndexMetrics = opts.IndexMetrics
+	serverOptions.QOSQueue = opts.QOSQueue
 
-	rs, err := resource.NewResourceServer(opts)
-	if err != nil {
-		return nil, err
-	}
-
-	return rs, nil
+	return resource.NewResourceServer(serverOptions)
 }
 
 // isHighAvailabilityEnabled determines if high availability mode should