Background Services: Add wrapper to support startup order (#110225)

This commit is contained in:
Todd Treece
2025-09-10 09:06:25 +01:00
committed by GitHub
parent 60d26ac0ce
commit 7c242eeaef
17 changed files with 790 additions and 36 deletions
@@ -0,0 +1,24 @@
package adapter
import "github.com/grafana/grafana/pkg/modules"
const (
// BackgroundServices is an alias for any background service that is not explicitly listed in the dependency map.
// This includes most background services in the BackgroundServiceRegistry.
BackgroundServices = "background-services"
// Core is an alias for a set of services that must be running before most other services can start.
Core = "core"
)
// dependencyMap returns the module dependency relationships for the background service system.
// It defines the startup order and dependencies between different module groups.
// Background services are automatically added as dependencies to the BackgroundServices module
// unless they are explicitly listed in this map with custom dependencies.
func dependencyMap() map[string][]string {
return map[string][]string{
modules.GrafanaAPIServer: {},
Core: {modules.GrafanaAPIServer},
BackgroundServices: {Core},
}
}
@@ -0,0 +1,28 @@
package adapter
import (
"slices"
"testing"
"github.com/stretchr/testify/require"
)
func TestDependencyMap(t *testing.T) {
t.Run("ensure every module except `BackgroundServices` is a dependency of another module", func(t *testing.T) {
deps := dependencyMap()
for module := range deps {
// it's safe to ignore the `BackgroundServices` module
if module == BackgroundServices {
continue
}
found := false
for _, moduleDeps := range deps {
if slices.Contains(moduleDeps, module) {
found = true
break
}
}
require.True(t, found, "module %s is not a dependency of any other module", module)
}
})
}
@@ -0,0 +1,12 @@
// This package provides an adapter layer between Grafana's background service registry
// and dskit's module and service managers. It enables Grafana background services to integrate
// with dskit's well-defined service states (New → Starting → Running → Stopping → Terminated)
// and module initialization order, allowing them to benefit from:
//
// - Coordinated service initialization
// - Observable service states and health monitoring
// - Graceful shutdown with proper cleanup ordering
//
// Background services that don't already implement the dskit's NamedService interface are
// automatically wrapped with dskit's BasicService and registered with dskit's module Manager.
package adapter
@@ -0,0 +1,102 @@
package adapter
import (
"context"
"reflect"
"sync"
"github.com/grafana/dskit/modules"
"github.com/grafana/dskit/services"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
grafanamodules "github.com/grafana/grafana/pkg/modules"
"github.com/grafana/grafana/pkg/registry"
)
type managerAdapter struct {
reg registry.BackgroundServiceRegistry
manager grafanamodules.Engine
mu sync.RWMutex // protects manager field from concurrent access
}
// NewManagerAdapter creates a new manager adapter that bridges Grafana's background
// service registry with dskit's module and service patterns. The adapter converts background
// services to dskit services and manages them using dskit's module Manager, which provides:
// - Coordinated service initialization
// - Observable service states and health monitoring
// - Graceful shutdown with proper cleanup ordering
func NewManagerAdapter(reg registry.BackgroundServiceRegistry) *managerAdapter {
return &managerAdapter{
reg: reg,
}
}
// Run initializes and starts all background services using dskit's module and service patterns.
//
// 1. Convert each registry.BackgroundService to a dskit service.NamedService (unless it already implements NamedService)
// 2. Register the services with the dskit module Manager
// 3. If the service is not already present in the dependency map, add it as a dependency of the `BackgroundServices` module
// 4. Initialize all services in the order of the dependency map
//
// Services implementing CanBeDisabled that are disabled will be skipped.
// The method blocks until the context is cancelled or a service fails.
func (r *managerAdapter) Run(ctx context.Context) error {
spanCtx, span := tracing.Start(ctx, "backgroundsvcs.adapter.Run")
defer span.End()
logger := log.New("backgroundsvcs.adapter").FromContext(spanCtx)
manager := modules.NewManager(logger)
deps := dependencyMap()
for _, bgSvc := range r.reg.GetServices() {
if s, ok := bgSvc.(registry.CanBeDisabled); ok && s.IsDisabled() {
logger.Debug("service is disabled, skipping", "service", reflect.TypeOf(bgSvc).String())
continue
}
namedService, ok := bgSvc.(services.NamedService)
if !ok {
// if the service is not a NamedService, try to convert it
namedService = asNamedService(bgSvc)
}
manager.RegisterModule(namedService.ServiceName(), func() (services.Service, error) {
return namedService, nil
}, modules.UserInvisibleModule)
// add the service as a background service dependency if it's not already in the dependency map
if _, ok := deps[namedService.ServiceName()]; !ok {
deps[namedService.ServiceName()] = []string{Core}
deps[BackgroundServices] = append(deps[BackgroundServices], namedService.ServiceName())
}
}
// any modules in the dependency map that haven't been registered should be registered.
// this should only include modules like all and core.
for modName := range deps {
if manager.IsModuleRegistered(modName) {
continue
}
logger.Debug("registering virtual module", "module", modName)
manager.RegisterModule(modName, nil)
}
r.mu.Lock()
r.manager = grafanamodules.NewWithManager(logger, []string{BackgroundServices}, manager, deps)
r.mu.Unlock()
logger.Debug("starting background services")
return r.manager.Run(spanCtx)
}
// Shutdown calls calls the underlying manager's Shutdown method if it has been initialized.
func (r *managerAdapter) Shutdown(ctx context.Context, reason string) error {
r.mu.RLock()
manager := r.manager
r.mu.RUnlock()
if manager == nil {
return nil
}
return manager.Shutdown(ctx, reason)
}
@@ -0,0 +1,207 @@
package adapter
import (
"context"
"testing"
"time"
"github.com/grafana/dskit/services"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/registry"
)
func TestNewManagerAdapter(t *testing.T) {
reg := &mockBackgroundServiceRegistry{}
adapter := NewManagerAdapter(reg)
require.NotNil(t, adapter)
require.Equal(t, reg, adapter.reg)
require.Nil(t, adapter.manager)
}
func TestManagerAdapter_Run(t *testing.T) {
t.Run("empty registry initializes manager", func(t *testing.T) {
reg := &mockBackgroundServiceRegistry{services: []registry.BackgroundService{}}
adapter := NewManagerAdapter(reg)
// Test that Run initializes the manager properly
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately to avoid hanging
err := adapter.Run(ctx)
require.NoError(t, err)
require.NotNil(t, adapter.manager)
})
t.Run("services are registered and called", func(t *testing.T) {
mockSvc := &mockService{}
// Make the service block until context is cancelled
mockSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
reg := &mockBackgroundServiceRegistry{services: []registry.BackgroundService{mockSvc}}
adapter := NewManagerAdapter(reg)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
err := adapter.Run(ctx)
require.NoError(t, err)
require.NotNil(t, adapter.manager)
require.True(t, mockSvc.runCalled)
})
}
func TestManagerAdapter_Run_ServiceTypes(t *testing.T) {
t.Run("service without NamedService interface gets converted", func(t *testing.T) {
mockSvc := &mockService{}
mockSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
reg := &mockBackgroundServiceRegistry{
services: []registry.BackgroundService{mockSvc},
}
adapter := NewManagerAdapter(reg)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
err := adapter.Run(ctx)
require.NoError(t, err)
require.NotNil(t, adapter.manager)
// Verify the service was called
require.True(t, mockSvc.runCalled)
})
t.Run("service with NamedService interface is used directly", func(t *testing.T) {
mockSvc := &mockNamedService{name: "custom-service"}
mockSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
reg := &mockBackgroundServiceRegistry{
services: []registry.BackgroundService{mockSvc},
}
adapter := NewManagerAdapter(reg)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
err := adapter.Run(ctx)
require.NoError(t, err)
require.NotNil(t, adapter.manager)
// Verify the service was called
require.True(t, mockSvc.runCalled)
})
t.Run("disabled service is skipped", func(t *testing.T) {
disabledSvc := &mockService{}
disabledSvc.disabled = true
enabledSvc := &mockService{}
enabledSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
reg := &mockBackgroundServiceRegistry{
services: []registry.BackgroundService{disabledSvc, enabledSvc},
}
adapter := NewManagerAdapter(reg)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
err := adapter.Run(ctx)
require.NoError(t, err)
require.NotNil(t, adapter.manager)
// Verify only enabled service was called
require.False(t, disabledSvc.runCalled)
require.True(t, enabledSvc.runCalled)
})
}
func TestManagerAdapter_Shutdown(t *testing.T) {
t.Run("shutdown with nil manager returns nil", func(t *testing.T) {
reg := &mockBackgroundServiceRegistry{}
adapter := NewManagerAdapter(reg)
err := adapter.Shutdown(context.Background(), "test shutdown")
require.NoError(t, err)
})
t.Run("shutdown with initialized manager", func(t *testing.T) {
mockSvc := &mockService{}
mockSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
reg := &mockBackgroundServiceRegistry{
services: []registry.BackgroundService{mockSvc},
}
adapter := NewManagerAdapter(reg)
// Initialize the manager
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
go func() {
err := adapter.Run(ctx)
require.NoError(t, err)
}()
// Give it a moment to initialize, then test shutdown
require.Eventually(t, func() bool {
return adapter.manager != nil
}, testContextTimeout, 5*time.Millisecond)
err := adapter.Shutdown(context.Background(), "test shutdown")
require.NoError(t, err)
})
}
func TestManagerAdapter_MultipleServices(t *testing.T) {
t.Run("multiple services are registered", func(t *testing.T) {
mockSvc1 := &mockService{}
mockSvc2 := &mockService{}
reg := &mockBackgroundServiceRegistry{
services: []registry.BackgroundService{mockSvc1, mockSvc2},
}
adapter := NewManagerAdapter(reg)
require.NotNil(t, adapter)
require.Equal(t, reg, adapter.reg)
require.Len(t, reg.GetServices(), 2)
})
}
type mockBackgroundServiceRegistry struct {
services []registry.BackgroundService
}
func (m *mockBackgroundServiceRegistry) GetServices() []registry.BackgroundService {
return m.services
}
type mockNamedService struct {
mockService
name string
}
func (m *mockNamedService) ServiceName() string {
return m.name
}
func (m *mockNamedService) State() services.State {
return services.New
}
@@ -0,0 +1,58 @@
package adapter
import (
"context"
"errors"
"reflect"
"github.com/grafana/dskit/services"
"github.com/grafana/grafana/pkg/registry"
)
var _ services.NamedService = &serviceAdapter{}
// serviceAdapter adapts a Grafana background service to dskit's NamedService interface.
// It wraps the background service with dskit's BasicService to provide the standard
// service state model: New → Starting → Running → Stopping → Terminated/Failed.
//
// The adapter uses dskit's BasicService with a custom RunningFn:
// - Starting phase: No-op, transitions immediately to Running
// - Running phase: Delegates to the wrapped service's Run method
// - Stopping phase: No-op, transitions immediately to Terminated/Failed
type serviceAdapter struct {
*services.BasicService
name string
service registry.BackgroundService
}
// asNamedService converts a Grafana background service into a dskit NamedService.
// The returned service starts in the New state and can be managed using dskit's
// standard service operations (StartAsync, AwaitRunning, StopAsync, AwaitTerminated).
//
// The service name is derived from the Go type name using reflection, ensuring
// each service type has a unique identifier within the dskit module system.
func asNamedService(service registry.BackgroundService) *serviceAdapter {
name := reflect.TypeOf(service).String()
a := &serviceAdapter{
name: name,
service: service,
}
a.BasicService = services.NewBasicService(nil, a.run, nil).WithName(name)
return a
}
// run implements the RunningFn for dskit's BasicService.
// This phase keeps the service in Running state by delegating to the wrapped
// background service's Run method. If the background service completes without
// error, the adapter waits for context cancellation (service stop) before
// transitioning to Stopping state, ensuring proper dskit service lifecycle.
func (a *serviceAdapter) run(ctx context.Context) error {
err := a.service.Run(ctx)
if err != nil && !errors.Is(err, context.Canceled) {
return err
}
// wait for context cancellation to transition to Stopping state.
// this prevents the service from causing it's dependents to stop prematurely.
<-ctx.Done()
return nil
}
@@ -0,0 +1,240 @@
package adapter
import (
"context"
"errors"
"reflect"
"testing"
"time"
"github.com/grafana/dskit/services"
"github.com/grafana/grafana/pkg/registry"
"github.com/stretchr/testify/require"
)
const (
// Constants for timeout-based tests
testContextTimeout = 50 * time.Millisecond
expectedMinDuration = 45 * time.Millisecond
)
func TestAsNamedService(t *testing.T) {
t.Run("creates service adapter with correct name", func(t *testing.T) {
mockSvc := &mockService{}
adapter := asNamedService(mockSvc)
require.NotNil(t, adapter)
require.NotNil(t, adapter.BasicService)
expectedName := reflect.TypeOf(mockSvc).String()
require.Equal(t, expectedName, adapter.name)
require.Equal(t, expectedName, adapter.ServiceName())
require.Equal(t, mockSvc, adapter.service)
})
t.Run("implements NamedService interface", func(t *testing.T) {
mockSvc := &mockService{}
adapter := asNamedService(mockSvc)
// Verify it implements the interface
require.Implements(t, (*services.NamedService)(nil), adapter)
// Verify it has the expected methods
require.NotEmpty(t, adapter.ServiceName())
require.Equal(t, services.New, adapter.State())
})
t.Run("creates BasicService", func(t *testing.T) {
mockSvc := &mockService{}
adapter := asNamedService(mockSvc)
require.NotNil(t, adapter.BasicService)
})
}
func TestServiceAdapter_Run(t *testing.T) {
t.Run("run calls underlying service and waits for context", func(t *testing.T) {
mockSvc := &mockService{}
mockSvc.runFunc = func(ctx context.Context) error {
// Simulate service running until context is cancelled
<-ctx.Done()
return nil
}
adapter := asNamedService(mockSvc)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
err := adapter.run(ctx)
require.NoError(t, err)
require.True(t, mockSvc.runCalled)
})
t.Run("run returns error from underlying service", func(t *testing.T) {
expectedErr := errors.New("service error")
mockSvc := &mockService{}
mockSvc.runError = expectedErr
adapter := asNamedService(mockSvc)
err := adapter.run(context.Background())
require.Error(t, err)
require.Equal(t, expectedErr, err)
require.True(t, mockSvc.runCalled)
})
t.Run("run waits for context cancellation after service completes", func(t *testing.T) {
mockSvc := &mockService{}
// Service completes immediately, adapter should wait for context
adapter := asNamedService(mockSvc)
ctx, cancel := context.WithTimeout(context.Background(), testContextTimeout)
defer cancel()
start := time.Now()
err := adapter.run(ctx)
duration := time.Since(start)
require.NoError(t, err)
require.GreaterOrEqual(t, duration, expectedMinDuration) // Should wait for context timeout
require.True(t, mockSvc.runCalled)
})
t.Run("run with immediately cancelled context", func(t *testing.T) {
mockSvc := &mockService{}
// Service completes immediately
adapter := asNamedService(mockSvc)
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
err := adapter.run(ctx)
require.NoError(t, err)
require.True(t, mockSvc.runCalled)
})
}
func TestServiceAdapter_Integration(t *testing.T) {
t.Run("full lifecycle with BasicService", func(t *testing.T) {
mockSvc := &mockService{}
mockSvc.runFunc = func(ctx context.Context) error {
<-ctx.Done()
return nil
}
adapter := asNamedService(mockSvc)
// Test that we can start the service
require.Equal(t, services.New, adapter.State())
// The BasicService should be properly configured
require.NotNil(t, adapter.BasicService)
require.Contains(t, adapter.ServiceName(), "mockService")
require.False(t, mockSvc.runCalled)
})
t.Run("service name reflects actual type", func(t *testing.T) {
// Test with different service types
mockSvc1 := &mockService{}
adapter1 := asNamedService(mockSvc1)
// Create a different type for comparison
type anotherMockService struct{ mockService }
mockSvc2 := &anotherMockService{}
adapter2 := asNamedService(mockSvc2)
require.Contains(t, adapter1.ServiceName(), "mockService")
require.Contains(t, adapter2.ServiceName(), "anotherMockService")
require.NotEqual(t, adapter1.ServiceName(), adapter2.ServiceName())
})
}
func TestServiceAdapter_ErrorHandling(t *testing.T) {
t.Run("generic error", func(t *testing.T) {
expectedErr := errors.New("generic error")
mockSvc := &mockService{}
mockSvc.runError = expectedErr
adapter := asNamedService(mockSvc)
t.Cleanup(func() {
adapter.StopAsync()
err := adapter.AwaitTerminated(context.Background())
require.ErrorIs(t, err, expectedErr)
})
err := adapter.StartAsync(context.Background())
require.NoError(t, err)
err = adapter.AwaitRunning(context.Background())
require.ErrorIs(t, err, expectedErr)
require.True(t, mockSvc.runCalled)
})
t.Run("context error", func(t *testing.T) {
mockSvc := &mockService{}
mockSvc.runError = context.Canceled
adapter := asNamedService(mockSvc)
t.Cleanup(func() {
adapter.StopAsync()
err := adapter.AwaitTerminated(context.Background())
require.NoError(t, err)
})
err := adapter.StartAsync(context.Background())
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
cancel()
err = adapter.AwaitRunning(ctx)
require.ErrorIs(t, err, context.Canceled)
})
t.Run("timeout error", func(t *testing.T) {
expectedErr := context.DeadlineExceeded
mockSvc := &mockService{}
mockSvc.runError = expectedErr
adapter := asNamedService(mockSvc)
t.Cleanup(func() {
adapter.StopAsync()
err := adapter.AwaitTerminated(context.Background())
require.ErrorIs(t, err, expectedErr)
})
err := adapter.StartAsync(context.Background())
require.NoError(t, err)
err = adapter.AwaitRunning(context.Background())
require.ErrorIs(t, err, expectedErr)
require.True(t, mockSvc.runCalled)
})
}
var _ registry.CanBeDisabled = &mockService{}
var _ registry.BackgroundService = &mockService{}
type mockService struct {
runFunc func(ctx context.Context) error
runCalled bool
runContext context.Context
runError error
disabled bool
}
func (m *mockService) Run(ctx context.Context) error {
m.runCalled = true
m.runContext = ctx
if m.runFunc != nil {
return m.runFunc(ctx)
}
return m.runError
}
func (m *mockService) IsDisabled() bool {
return m.disabled
}