public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(unified-storage): add qos support for the resource server (#105939)

Browse Source
This commit is contained in:
Mustafa Sencer Özcan
2025-07-01 09:22:55 +00:00
committed by GitHub
parent 71a4f20770
commit 974a2c47f9
12 changed files with 397 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/sql/server.go
+50 -33
View File
@@ -1,6 +1,7 @@
package sql
import (
"context"
"os"
"strings"
@@ -8,6 +9,7 @@ import (
"go.opentelemetry.io/otel/trace"
"github.com/grafana/authlib/types"
"github.com/grafana/dskit/services"
infraDB "github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -17,70 +19,85 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
)
type QOSEnqueueDequeuer interface {
services.Service
Enqueue(ctx context.Context, tenantID string, runnable func(ctx context.Context)) error
Dequeue(ctx context.Context) (func(ctx context.Context), error)
}
// ServerOptions contains the options for creating a new ResourceServer
type ServerOptions struct {
DB infraDB.DB
Cfg *setting.Cfg
Tracer trace.Tracer
Reg prometheus.Registerer
AccessClient types.AccessClient
SearchOptions resource.SearchOptions
StorageMetrics *resource.StorageMetrics
IndexMetrics *resource.BleveIndexMetrics
Features featuremgmt.FeatureToggles
QOSQueue QOSEnqueueDequeuer
}
// Creates a new ResourceServer
func NewResourceServer(db infraDB.DB, cfg *setting.Cfg,
tracer trace.Tracer, reg prometheus.Registerer, ac types.AccessClient,
searchOptions resource.SearchOptions, storageMetrics *resource.StorageMetrics,
indexMetrics *resource.BleveIndexMetrics, features featuremgmt.FeatureToggles) (resource.ResourceServer, error) {
apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver")
opts := resource.ResourceServerOptions{
Tracer: tracer,
func NewResourceServer(
opts ServerOptions,
) (resource.ResourceServer, error) {
apiserverCfg := opts.Cfg.SectionWithEnvOverrides("grafana-apiserver")
serverOptions := resource.ResourceServerOptions{
Tracer: opts.Tracer,
Blob: resource.BlobConfig{
URL: apiserverCfg.Key("blob_url").MustString(""),
},
Reg: reg,
Reg: opts.Reg,
}
if ac != nil {
opts.AccessClient = resource.NewAuthzLimitedClient(ac, resource.AuthzOptions{Tracer: tracer, Registry: reg})
if opts.AccessClient != nil {
serverOptions.AccessClient = resource.NewAuthzLimitedClient(opts.AccessClient, resource.AuthzOptions{Tracer: opts.Tracer, Registry: opts.Reg})
}
// Support local file blob
if strings.HasPrefix(opts.Blob.URL, "./data/") {
dir := strings.Replace(opts.Blob.URL, "./data", cfg.DataPath, 1)
if strings.HasPrefix(serverOptions.Blob.URL, "./data/") {
dir := strings.Replace(serverOptions.Blob.URL, "./data", opts.Cfg.DataPath, 1)
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, err
}
opts.Blob.URL = "file:///" + dir
serverOptions.Blob.URL = "file:///" + dir
}
// This is mostly for testing, being able to influence when we paginate
// based on the page size during tests.
unifiedStorageCfg := cfg.SectionWithEnvOverrides("unified_storage")
unifiedStorageCfg := opts.Cfg.SectionWithEnvOverrides("unified_storage")
maxPageSizeBytes := unifiedStorageCfg.Key("max_page_size_bytes")
opts.MaxPageSizeBytes = maxPageSizeBytes.MustInt(0)
serverOptions.MaxPageSizeBytes = maxPageSizeBytes.MustInt(0)
eDB, err := dbimpl.ProvideResourceDB(db, cfg, tracer)
eDB, err := dbimpl.ProvideResourceDB(opts.DB, opts.Cfg, opts.Tracer)
if err != nil {
return nil, err
}
isHA := isHighAvailabilityEnabled(cfg.SectionWithEnvOverrides("database"),
cfg.SectionWithEnvOverrides("resource_api"))
withPruner := features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageHistoryPruner)
isHA := isHighAvailabilityEnabled(opts.Cfg.SectionWithEnvOverrides("database"),
opts.Cfg.SectionWithEnvOverrides("resource_api"))
withPruner := opts.Features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageHistoryPruner)
store, err := NewBackend(BackendOptions{
DBProvider: eDB,
Tracer: tracer,
Reg: reg,
Tracer: opts.Tracer,
Reg: opts.Reg,
IsHA: isHA,
withPruner: withPruner,
storageMetrics: storageMetrics,
storageMetrics: opts.StorageMetrics,
})
if err != nil {
return nil, err
}
opts.Backend = store
opts.Diagnostics = store
opts.Lifecycle = store
opts.Search = searchOptions
opts.IndexMetrics = indexMetrics
serverOptions.Backend = store
serverOptions.Diagnostics = store
serverOptions.Lifecycle = store
serverOptions.Search = opts.SearchOptions
serverOptions.IndexMetrics = opts.IndexMetrics
serverOptions.QOSQueue = opts.QOSQueue
rs, err := resource.NewResourceServer(opts)
if err != nil {
return nil, err
}
return rs, nil
return resource.NewResourceServer(serverOptions)
}
// isHighAvailabilityEnabled determines if high availability mode should
pkg/storage/unified/sql/service.go
+87 -29
View File
@@ -34,6 +34,7 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/storage/unified/search"
"github.com/grafana/grafana/pkg/util/scheduler"
)
var (
@@ -50,6 +51,11 @@ type UnifiedStorageGrpcService interface {
type service struct {
*services.BasicService
// Subservices manager
subservices *services.Manager
subservicesWatcher *services.FailureWatcher
hasSubservices bool
cfg *setting.Cfg
features featuremgmt.FeatureToggles
db infraDB.DB
@@ -71,6 +77,9 @@ type service struct {
storageRing *ring.Ring
lifecycler *ring.BasicLifecycler
queue QOSEnqueueDequeuer
scheduler *scheduler.Scheduler
}
func ProvideUnifiedStorageGrpcService(
@@ -85,6 +94,7 @@ func ProvideUnifiedStorageGrpcService(
storageRing *ring.Ring,
memberlistKVConfig kv.Config,
) (UnifiedStorageGrpcService, error) {
var err error
tracer := otel.Tracer("unified-storage")
// FIXME: This is a temporary solution while we are migrating to the new authn interceptor
@@ -95,20 +105,22 @@ func ProvideUnifiedStorageGrpcService(
})
s := &service{
cfg: cfg,
features: features,
stopCh: make(chan struct{}),
authenticator: authn,
tracing: tracer,
db: db,
log: log,
reg: reg,
docBuilders: docBuilders,
storageMetrics: storageMetrics,
indexMetrics: indexMetrics,
storageRing: storageRing,
cfg: cfg,
features: features,
stopCh: make(chan struct{}),
authenticator: authn,
tracing: tracer,
db: db,
log: log,
reg: reg,
docBuilders: docBuilders,
storageMetrics: storageMetrics,
indexMetrics: indexMetrics,
storageRing: storageRing,
subservicesWatcher: services.NewFailureWatcher(),
}
subservices := []services.Service{}
if cfg.EnableSharding {
ringStore, err := kv.NewClient(
memberlistKVConfig,
@@ -143,15 +155,50 @@ func ProvideUnifiedStorageGrpcService(
if err != nil {
return nil, fmt.Errorf("failed to initialize storage-ring lifecycler: %s", err)
}
subservices = append(subservices, s.lifecycler)
}
if cfg.QOSEnabled {
qosReg := prometheus.WrapRegistererWithPrefix("resource_server_qos_", reg)
queue := scheduler.NewQueue(&scheduler.QueueOptions{
MaxSizePerTenant: cfg.QOSMaxSizePerTenant,
Registerer: qosReg,
})
scheduler, err := scheduler.NewScheduler(queue, &scheduler.Config{
NumWorkers: cfg.QOSNumberWorker,
Logger: log,
})
if err != nil {
return nil, fmt.Errorf("failed to create qos scheduler: %s", err)
}
s.queue = queue
s.scheduler = scheduler
subservices = append(subservices, s.queue, s.scheduler)
}
if len(subservices) > 0 {
s.hasSubservices = true
s.subservices, err = services.NewManager(subservices...)
if err != nil {
return nil, fmt.Errorf("failed to create subservices manager: %w", err)
}
}
// This will be used when running as a dskit service
s.BasicService = services.NewBasicService(s.start, s.running, s.stopping).WithName(modules.StorageServer)
s.BasicService = services.NewBasicService(s.starting, s.running, s.stopping).WithName(modules.StorageServer)
return s, nil
}
func (s *service) start(ctx context.Context) error {
func (s *service) starting(ctx context.Context) error {
if s.hasSubservices {
s.subservicesWatcher.WatchManager(s.subservices)
if err := services.StartManagerAndAwaitHealthy(ctx, s.subservices); err != nil {
return fmt.Errorf("failed to start subservices: %w", err)
}
}
authzClient, err := authz.ProvideStandaloneAuthZClient(s.cfg, s.features, s.tracing)
if err != nil {
return err
@@ -162,7 +209,19 @@ func (s *service) start(ctx context.Context) error {
return err
}
server, err := NewResourceServer(s.db, s.cfg, s.tracing, s.reg, authzClient, searchOptions, s.storageMetrics, s.indexMetrics, s.features)
serverOptions := ServerOptions{
DB: s.db,
Cfg: s.cfg,
Tracer: s.tracing,
Reg: s.reg,
AccessClient: authzClient,
SearchOptions: searchOptions,
StorageMetrics: s.storageMetrics,
IndexMetrics: s.indexMetrics,
Features: s.features,
QOSQueue: s.queue,
}
server, err := NewResourceServer(serverOptions)
if err != nil {
return err
}
@@ -192,11 +251,6 @@ func (s *service) start(ctx context.Context) error {
}
if s.cfg.EnableSharding {
err = s.lifecycler.StartAsync(ctx)
if err != nil {
return fmt.Errorf("failed to start the lifecycler: %s", err)
}
s.log.Info("waiting until resource server is JOINING in the ring")
lfcCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
@@ -231,15 +285,27 @@ func (s *service) GetAddress() string {
func (s *service) running(ctx context.Context) error {
select {
case err := <-s.stoppedCh:
if err != nil {
if err != nil && !errors.Is(err, context.Canceled) {
return err
}
case err := <-s.subservicesWatcher.Chan():
return fmt.Errorf("subservice failure: %w", err)
case <-ctx.Done():
close(s.stopCh)
}
return nil
}
func (s *service) stopping(_ error) error {
if s.hasSubservices {
err := services.StopManagerAndAwaitStopped(context.Background(), s.subservices)
if err != nil {
return fmt.Errorf("failed to stop subservices: %w", err)
}
}
return nil
}
type authenticatorWithFallback struct {
authenticator func(ctx context.Context) (context.Context, error)
fallback func(ctx context.Context) (context.Context, error)
@@ -309,14 +375,6 @@ func NewAuthenticatorWithFallback(cfg *setting.Cfg, reg prometheus.Registerer, t
}
}
func (s *service) stopping(err error) error {
if err != nil && !errors.Is(err, context.Canceled) {
s.log.Error("stopping unified storage grpc service", "error", err)
return err
}
return nil
}
func toLifecyclerConfig(cfg *setting.Cfg, logger log.Logger) (ring.BasicLifecyclerConfig, error) {
instanceAddr, err := ring.GetInstanceAddr(cfg.MemberlistBindAddr, netutil.PrivateNetworkInterfacesWithFallback([]string{"eth0", "en0"}, logger), logger, true)
if err != nil {