Alerting: Instrument outbound requests for Loki Historian and Remote Alertmanager with tracing (#89185)

* Add TracedClient

* Handle errors and status codes

* Wire up tracing to normal ASH and loki annotation mapping

* Add tracing to remote alertmanager

* one more spot

* and not or

* More consistency with other grafana traces, lower cardinality name

pkg/services/annotations/annotationsimpl/annotations.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/annotations"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/tag"
@@ -27,6 +28,7 @@ func ProvideService(
 	cfg *setting.Cfg,
 	features featuremgmt.FeatureToggles,
 	tagService tag.Service,
+	tracer tracing.Tracer,
 ) *RepositoryImpl {
 	l := log.New("annotations")
 	l.Debug("Initializing annotations service")
@@ -35,7 +37,7 @@ func ProvideService(
 	write := xormStore
 
 	var read readStore
-	historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"))
+	historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"), tracer)
 	if historianStore != nil {
 		l.Debug("Using composite read store")
 		read = NewCompositeStore(log.New("annotations.composite"), xormStore, historianStore)

pkg/services/annotations/annotationsimpl/annotations_test.go

@@ -48,7 +48,7 @@ func TestIntegrationAnnotationListingWithRBAC(t *testing.T) {
 	features := featuremgmt.WithFeatures()
 	tagService := tagimpl.ProvideService(sql)
 
-	repo := ProvideService(sql, cfg, features, tagService)
+	repo := ProvideService(sql, cfg, features, tagService, tracing.InitializeTracerForTest())
 
 	dashboard1 := testutil.CreateDashboard(t, sql, cfg, features, dashboards.SaveDashboardCommand{
 		UserID:   1,
@@ -317,7 +317,7 @@ func TestIntegrationAnnotationListingWithInheritedRBAC(t *testing.T) {
 			cfg := setting.NewCfg()
 			cfg.AnnotationMaximumTagsLength = 60
 
-			repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql))
+			repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql), tracing.InitializeTracerForTest())
 
 			usr.Permissions = map[int64]map[string][]string{1: tc.permissions}
 			testutil.SetupRBACPermission(t, sql, role, usr)

pkg/services/annotations/annotationsimpl/loki/historian_store.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
@@ -53,7 +54,7 @@ type LokiHistorianStore struct {
 	log    log.Logger
 }
 
-func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger) *LokiHistorianStore {
+func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger, tracer tracing.Tracer) *LokiHistorianStore {
 	if !useStore(cfg, ft) {
 		return nil
 	}
@@ -64,7 +65,7 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft f
 	}
 
 	return &LokiHistorianStore{
-		client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log),
+		client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log, tracer),
 		db:     db,
 		log:    log,
 	}

pkg/services/ngalert/client/client.go

@@ -7,6 +7,11 @@ import (
 	"strconv"
 
 	"github.com/grafana/dskit/instrument"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // Requester executes an HTTP request.
@@ -14,7 +19,7 @@ type Requester interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
-// TimedClient instruments a request. It implements Requester.
+// TimedClient instruments a request with metrics. It implements Requester.
 type TimedClient struct {
 	client    Requester
 	collector instrument.Collector
@@ -70,3 +75,51 @@ func TimeRequest(ctx context.Context, operation string, coll instrument.Collecto
 		coll, toStatusCode, doRequest)
 	return response, err
 }
+
+// TracedClient instruments a request with tracing. It implements Requester.
+type TracedClient struct {
+	client Requester
+	tracer tracing.Tracer
+	name   string
+}
+
+func NewTracedClient(client Requester, tracer tracing.Tracer, name string) *TracedClient {
+	return &TracedClient{
+		client: client,
+		tracer: tracer,
+		name:   name,
+	}
+}
+
+// Do executes the request.
+func (c TracedClient) Do(r *http.Request) (*http.Response, error) {
+	ctx, span := c.tracer.Start(r.Context(), c.name, trace.WithSpanKind(trace.SpanKindClient))
+	defer span.End()
+
+	span.SetAttributes(semconv.HTTPURL(r.URL.String()))
+	span.SetAttributes(semconv.HTTPMethod(r.Method))
+
+	c.tracer.Inject(ctx, r.Header, span)
+
+	r = r.WithContext(ctx)
+	resp, err := c.client.Do(r)
+	if err != nil {
+		span.SetStatus(codes.Error, "request failed")
+		span.RecordError(err)
+	} else {
+		if resp.ContentLength > 0 {
+			span.SetAttributes(attribute.Int64("http.content_length", resp.ContentLength))
+		}
+		span.SetAttributes(semconv.HTTPStatusCode(resp.StatusCode))
+		if resp.StatusCode >= 400 && resp.StatusCode < 600 {
+			span.RecordError(fmt.Errorf("error with HTTP status code %d", resp.StatusCode))
+		}
+	}
+
+	return resp, err
+}
+
+// RoundTrip implements the RoundTripper interface.
+func (c TracedClient) RoundTrip(r *http.Request) (*http.Response, error) {
+	return c.Do(r)
+}

pkg/services/ngalert/ngalert.go

@@ -200,7 +200,7 @@ func (ng *AlertNG) init() error {
 						PromoteConfig:     true,
 						SyncInterval:      ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
 					}
-					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
+					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
 					if err != nil {
 						moaLogger.Error("Failed to create remote Alertmanager", "err", err)
 						return nil, err
@@ -234,7 +234,7 @@ func (ng *AlertNG) init() error {
 						TenantID:          ng.Cfg.UnifiedAlerting.RemoteAlertmanager.TenantID,
 						URL:               ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
 					}
-					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
+					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
 					if err != nil {
 						moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
 						return internalAM, nil
@@ -270,7 +270,7 @@ func (ng *AlertNG) init() error {
 						URL:               ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
 						SyncInterval:      ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
 					}
-					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
+					remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
 					if err != nil {
 						moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
 						return internalAM, nil
@@ -359,7 +359,7 @@ func (ng *AlertNG) init() error {
 	// There are a set of feature toggles available that act as short-circuits for common configurations.
 	// If any are set, override the config accordingly.
 	ApplyStateHistoryFeatureToggles(&ng.Cfg.UnifiedAlerting.StateHistory, ng.FeatureToggles, ng.Log)
-	history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log)
+	history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer)
 	if err != nil {
 		return err
 	}
@@ -523,7 +523,7 @@ type Historian interface {
 	state.Historian
 }
 
-func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger) (Historian, error) {
+func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger, tracer tracing.Tracer) (Historian, error) {
 	if !cfg.Enabled {
 		met.Info.WithLabelValues("noop").Set(0)
 		return historian.NewNopHistorian(), nil
@@ -538,7 +538,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
 	if backend == historian.BackendTypeMultiple {
 		primaryCfg := cfg
 		primaryCfg.Backend = cfg.MultiPrimary
-		primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l)
+		primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l, tracer)
 		if err != nil {
 			return nil, fmt.Errorf("multi-backend target \"%s\" was misconfigured: %w", cfg.MultiPrimary, err)
 		}
@@ -547,7 +547,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
 		for _, b := range cfg.MultiSecondaries {
 			secCfg := cfg
 			secCfg.Backend = b
-			sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l)
+			sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l, tracer)
 			if err != nil {
 				return nil, fmt.Errorf("multi-backend target \"%s\" was miconfigured: %w", b, err)
 			}
@@ -569,7 +569,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
 		}
 		req := historian.NewRequester()
 		lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
-		backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met)
+		backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met, tracer)
 
 		testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second)
 		defer cancelFunc()
@@ -627,8 +627,8 @@ func ApplyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySet
 	}
 }
 
-func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager) (*remote.Alertmanager, error) {
+func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*remote.Alertmanager, error) {
-	return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m)
+	return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
 }
 
 func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings) (schedule.RecordingWriter, error) {

pkg/services/ngalert/ngalert_test.go

@@ -62,12 +62,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
 	t.Run("fail initialization if invalid backend", func(t *testing.T) {
 		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled: true,
 			Backend: "invalid-backend",
 		}
 
-		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.ErrorContains(t, err, "unrecognized")
 	})
@@ -75,13 +76,14 @@ func TestConfigureHistorianBackend(t *testing.T) {
 	t.Run("fail initialization if invalid multi-backend primary", func(t *testing.T) {
 		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled:      true,
 			Backend:      "multiple",
 			MultiPrimary: "invalid-backend",
 		}
 
-		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.ErrorContains(t, err, "multi-backend target")
 		require.ErrorContains(t, err, "unrecognized")
@@ -90,6 +92,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
 	t.Run("fail initialization if invalid multi-backend secondary", func(t *testing.T) {
 		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled:          true,
 			Backend:          "multiple",
@@ -97,7 +100,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
 			MultiSecondaries: []string{"annotations", "invalid-backend"},
 		}
 
-		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.ErrorContains(t, err, "multi-backend target")
 		require.ErrorContains(t, err, "unrecognized")
@@ -106,6 +109,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
 	t.Run("do not fail initialization if pinging Loki fails", func(t *testing.T) {
 		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled: true,
 			Backend: "loki",
@@ -114,7 +118,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
 			LokiWriteURL: "http://gone.invalid",
 		}
 
-		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.NotNil(t, h)
 		require.NoError(t, err)
@@ -124,12 +128,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
 		reg := prometheus.NewRegistry()
 		met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled: true,
 			Backend: "annotations",
 		}
 
-		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.NotNil(t, h)
 		require.NoError(t, err)
@@ -146,11 +151,12 @@ grafana_alerting_state_history_info{backend="annotations"} 1
 		reg := prometheus.NewRegistry()
 		met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
 		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
 		cfg := setting.UnifiedAlertingStateHistorySettings{
 			Enabled: false,
 		}
 
-		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
+		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
 
 		require.NotNil(t, h)
 		require.NoError(t, err)

pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -66,7 +67,7 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
 				DefaultConfig:     setting.GetAlertmanagerDefaultConfiguration(),
 			}
 			m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-			remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m)
+			remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m, tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 
 			// Use both Alertmanager implementations in the forked Alertmanager.

pkg/services/ngalert/remote/alertmanager.go

@@ -25,6 +25,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -100,7 +101,7 @@ func (cfg *AlertmanagerConfig) Validate() error {
 	return nil
 }
 
-func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
+func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
 	if err := cfg.Validate(); err != nil {
 		return nil, err
 	}
@@ -118,7 +119,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
 		URL:           u,
 		PromoteConfig: cfg.PromoteConfig,
 	}
-	mc, err := remoteClient.New(mcCfg, metrics)
+	mc, err := remoteClient.New(mcCfg, metrics, tracer)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +130,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
 		Password: cfg.BasicAuthPassword,
 		Logger:   logger,
 	}
-	amc, err := remoteClient.NewAlertmanager(amcCfg, metrics)
+	amc, err := remoteClient.NewAlertmanager(amcCfg, metrics, tracer)
 	if err != nil {
 		return nil, err
 	}

pkg/services/ngalert/remote/alertmanager_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/grafana/alerting/definition"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -105,7 +106,7 @@ func TestNewAlertmanager(t *testing.T) {
 				DefaultConfig:     defaultGrafanaConfig,
 			}
 			m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-			am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
+			am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 			if test.expErr != "" {
 				require.EqualError(tt, err, test.expErr)
 				return
@@ -177,7 +178,7 @@ func TestApplyConfig(t *testing.T) {
 
 	// An error response from the remote Alertmanager should result in the readiness check failing.
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	config := &ngmodels.AlertConfiguration{
@@ -305,6 +306,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 				decryptFn,
 				test.autogenFn,
 				m,
+				tracing.InitializeTracerForTest(),
 			)
 			require.NoError(t, err)
 
@@ -364,7 +366,7 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 
 	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	encodedFullState, err := am.getFullState(ctx)
@@ -521,7 +523,7 @@ func TestIntegrationRemoteAlertmanagerGetStatus(t *testing.T) {
 
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	// We should get the default Cloud Alertmanager configuration.
@@ -555,7 +557,7 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
 
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	// We should have no silences at first.
@@ -640,7 +642,7 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	// Wait until the Alertmanager is ready to send alerts.
@@ -709,7 +711,7 @@ func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {
 
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
-	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
+	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
 	require.NoError(t, err)
 
 	// We should start with the default config.

pkg/services/ngalert/remote/client/alertmanager.go

@@ -9,6 +9,7 @@ import (
 
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	amclient "github.com/prometheus/alertmanager/api/v2/client"
@@ -31,7 +32,7 @@ type Alertmanager struct {
 	logger     log.Logger
 }
 
-func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
+func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
 	// First, add the authentication middleware.
 	c := &http.Client{Transport: &MimirAuthRoundTripper{
 		TenantID: cfg.TenantID,
@@ -40,14 +41,15 @@ func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanage
 	}}
 
 	tc := client.NewTimedClient(c, metrics.RequestLatency)
+	trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
 	apiEndpoint := *cfg.URL
 
 	// Next, make sure you set the right path.
 	u := apiEndpoint.JoinPath(alertmanagerAPIMountPath, amclient.DefaultBasePath)
 
-	// Create an Alertmanager client using the timed client as the transport.
+	// Create an Alertmanager client using the instrumented client as the transport.
 	r := httptransport.New(u.Host, u.Path, []string{u.Scheme})
-	r.Transport = tc
+	r.Transport = trc
 
 	return &Alertmanager{
 		logger:          cfg.Logger,

pkg/services/ngalert/remote/client/mimir.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -68,7 +69,7 @@ func (e *errorResponse) Error() string {
 	return e.Error2
 }
 
-func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
+func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Mimir, error) {
 	rt := &MimirAuthRoundTripper{
 		TenantID: cfg.TenantID,
 		Password: cfg.Password,
@@ -78,10 +79,12 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
 	c := &http.Client{
 		Transport: rt,
 	}
+	tc := client.NewTimedClient(c, metrics.RequestLatency)
+	trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
 
 	return &Mimir{
 		endpoint:      cfg.URL,
-		client:        client.NewTimedClient(c, metrics.RequestLatency),
+		client:        trc,
 		logger:        cfg.Logger,
 		metrics:       metrics,
 		promoteConfig: cfg.PromoteConfig,

pkg/services/ngalert/state/historian/loki.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -55,9 +56,9 @@ type RemoteLokiBackend struct {
 	log            log.Logger
 }
 
-func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend {
+func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian, tracer tracing.Tracer) *RemoteLokiBackend {
 	return &RemoteLokiBackend{
-		client:         NewLokiClient(cfg, req, metrics, logger),
+		client:         NewLokiClient(cfg, req, metrics, logger, tracer),
 		externalLabels: cfg.ExternalLabels,
 		clock:          clock.New(),
 		metrics:        metrics,

pkg/services/ngalert/state/historian/loki_http.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/setting"
@@ -103,10 +104,11 @@ const (
 	NeqRegEx Operator = "!~"
 )
 
-func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *HttpLokiClient {
+func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger, tracer tracing.Tracer) *HttpLokiClient {
 	tc := client.NewTimedClient(req, metrics.WriteDuration)
+	trc := client.NewTracedClient(tc, tracer, "ngalert.historian.client")
 	return &HttpLokiClient{
-		client:  tc,
+		client:  trc,
 		encoder: cfg.Encoder,
 		cfg:     cfg,
 		metrics: metrics,

pkg/services/ngalert/state/historian/loki_http_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 )
 
 func TestLokiConfig(t *testing.T) {
@@ -227,7 +228,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
 			ReadPathURL:  url,
 			WritePathURL: url,
 			Encoder:      JsonEncoder{},
-		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
+		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
 
 		// Unauthorized request should fail against Grafana Cloud.
 		err = client.Ping(context.Background())
@@ -255,7 +256,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
 			BasicAuthUser:     "<your_username>",
 			BasicAuthPassword: "<your_password>",
 			Encoder:           JsonEncoder{},
-		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
+		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
 
 		// When running on prem, you might need to set the tenant id,
 		// so the x-scope-orgid header is set.
@@ -389,7 +390,7 @@ func createTestLokiClient(req client.Requester) *HttpLokiClient {
 		Encoder:      JsonEncoder{},
 	}
 	met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
-	return NewLokiClient(cfg, req, met, log.NewNopLogger())
+	return NewLokiClient(cfg, req, met, log.NewNopLogger(), tracing.InitializeTracerForTest())
 }
 
 func reqBody(t *testing.T, req *http.Request) string {

pkg/services/ngalert/state/historian/loki_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -514,7 +515,7 @@ func createTestLokiBackend(req client.Requester, met *metrics.Historian) *Remote
 		ExternalLabels: map[string]string{"externalLabelKey": "externalLabelValue"},
 	}
 	lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
-	return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met)
+	return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met, tracing.InitializeTracerForTest())
 }
 
 func singleFromNormal(st *state.State) []state.StateTransition {

pkg/services/publicdashboards/service/common_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/annotations"
 	"github.com/grafana/grafana/pkg/services/annotations/annotationsimpl"
 	"github.com/grafana/grafana/pkg/services/dashboards"
@@ -29,7 +30,7 @@ func newPublicDashboardServiceImpl(
 	db, cfg := db.InitTestDBWithCfg(t)
 	tagService := tagimpl.ProvideService(db)
 	if annotationsRepo == nil {
-		annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService)
+		annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService, tracing.InitializeTracerForTest())
 	}
 
 	if publicDashboardStore == nil {