Tracing: Standardize on otel tracing (#75528)

2023-10-03 14:54:20 +02:00
parent 4432c4c75c
commit e4c1a7a141
46 changed files with 321 additions and 439 deletions
@@ -9,6 +9,7 @@ import (
 	"github.com/benbjohnson/clock"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	"golang.org/x/sync/errgroup"

 	"github.com/grafana/grafana/pkg/infra/localcache"
@@ -206,13 +207,8 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 		defer func() {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
+				span.SetStatus(codes.Error, "failed to execute alert rule. panic was recovered.")
 				span.RecordError(fmt.Errorf("%v", err))
-				span.AddEvents(
-					[]string{"error", "message"},
-					[]tracing.EventValue{
-						{Str: fmt.Sprintf("%v", err)},
-						{Str: "failed to execute alert rule. panic was recovered."},
-					})
 				span.End()
 				close(attemptChan)
 			}
@@ -220,20 +216,17 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan

 		e.evalHandler.Eval(evalContext)

-		span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID))
-		span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID))
-		span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing))
-		span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound))
-		span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID))
+		span.SetAttributes(
+			attribute.Int64("alertId", evalContext.Rule.ID),
+			attribute.Int64("dashboardId", evalContext.Rule.DashboardID),
+			attribute.Bool("firing", evalContext.Firing),
+			attribute.Bool("nodatapoints", evalContext.NoDataFound),
+			attribute.Int("attemptID", attemptID),
+		)

 		if evalContext.Error != nil {
+			span.SetStatus(codes.Error, "alerting execution attempt failed")
 			span.RecordError(evalContext.Error)
-			span.AddEvents(
-				[]string{"error", "message"},
-				[]tracing.EventValue{
-					{Str: fmt.Sprintf("%v", evalContext.Error)},
-					{Str: "alerting execution attempt failed"},
-				})

 			if attemptID < setting.AlertingMaxAttempts {
 				span.End()
@@ -8,6 +8,7 @@ import (

 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/network"
@@ -262,9 +263,10 @@ func (s *Service) RegisterPostAuthHook(hook authn.PostAuthHookFn, priority uint)
 }

 func (s *Service) Login(ctx context.Context, client string, r *authn.Request) (identity *authn.Identity, err error) {
-	ctx, span := s.tracer.Start(ctx, "authn.Login")
+	ctx, span := s.tracer.Start(ctx, "authn.Login", trace.WithAttributes(
+		attribute.String(attributeKeyClient, client),
+	))
 	defer span.End()
-	span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))

 	defer func() {
 		for _, hook := range s.postLoginHooks.items {
@@ -316,9 +318,10 @@ func (s *Service) RegisterPostLoginHook(hook authn.PostLoginHookFn, priority uin
 }

 func (s *Service) RedirectURL(ctx context.Context, client string, r *authn.Request) (*authn.Redirect, error) {
-	ctx, span := s.tracer.Start(ctx, "authn.RedirectURL")
+	ctx, span := s.tracer.Start(ctx, "authn.RedirectURL", trace.WithAttributes(
+		attribute.String(attributeKeyClient, client),
+	))
 	defer span.End()
-	span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))

 	c, ok := s.clients[client]
 	if !ok {
@@ -138,7 +138,7 @@ func (srv *CleanUpService) cleanUpTmpFiles(ctx context.Context) {

 	for _, f := range folders {
 		ctx, span := srv.tracer.Start(ctx, "delete stale files in temporary directory")
-		span.SetAttributes("directory", f, attribute.Key("directory").String(f))
+		span.SetAttributes(attribute.String("directory", f))
 		srv.cleanUpTmpFolder(ctx, f)
 		span.End()
 	}
@@ -18,6 +18,8 @@ import (
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )

 func ProvideService(cfg *setting.Cfg, tracer tracing.Tracer, features *featuremgmt.FeatureManager, authnService authn.Service,
@@ -125,13 +127,11 @@ func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
 		}

 		reqContext.Logger = reqContext.Logger.New("userId", reqContext.UserID, "orgId", reqContext.OrgID, "uname", reqContext.Login)
-		span.AddEvents(
-			[]string{"uname", "orgId", "userId"},
-			[]tracing.EventValue{
-				{Str: reqContext.Login},
-				{Num: reqContext.OrgID},
-				{Num: reqContext.UserID}},
-		)
+		span.AddEvent("user", trace.WithAttributes(
+			attribute.String("uname", reqContext.Login),
+			attribute.Int64("orgId", reqContext.OrgID),
+			attribute.Int64("userId", reqContext.UserID),
+		))

 		next.ServeHTTP(w, r)
 	})
@@ -14,7 +14,7 @@ import (
 	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/services/auth/identity"
@@ -9,6 +9,8 @@ import (

 	"github.com/benbjohnson/clock"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"

 	"github.com/grafana/grafana/pkg/infra/log"
@@ -372,7 +374,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 		notify(states)
 	}

-	evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span tracing.Span) {
+	evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span trace.Span) {
 		logger := logger.New("version", e.rule.Version, "fingerprint", f, "attempt", attempt, "now", e.scheduledAt).FromContext(ctx)
 		start := sch.clock.Now()

@@ -406,21 +408,13 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 					}
 				}
 			}
+			span.SetStatus(codes.Error, "rule evaluation failed")
 			span.RecordError(err)
-			span.AddEvents(
-				[]string{"error", "message"},
-				[]tracing.EventValue{
-					{Str: fmt.Sprintf("%v", err)},
-					{Str: "rule evaluation failed"},
-				})
 		} else {
 			logger.Debug("Alert rule evaluated", "results", results, "duration", dur)
-			span.AddEvents(
-				[]string{"message", "results"},
-				[]tracing.EventValue{
-					{Str: "rule evaluated"},
-					{Num: int64(len(results))},
-				})
+			span.AddEvent("rule evaluated", trace.WithAttributes(
+				attribute.Int64("results", int64(len(results))),
+			))
 		}
 		if ctx.Err() != nil { // check if the context is not cancelled. The evaluation can be a long-running task.
 			logger.Debug("Skip updating the state because the context has been cancelled")
@@ -438,13 +432,10 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR

 		start = sch.clock.Now()
 		alerts := state.FromStateTransitionToPostableAlerts(processedStates, sch.stateManager, sch.appURL)
-		span.AddEvents(
-			[]string{"message", "state_transitions", "alerts_to_send"},
-			[]tracing.EventValue{
-				{Str: "results processed"},
-				{Num: int64(len(processedStates))},
-				{Num: int64(len(alerts.PostableAlerts))},
-			})
+		span.AddEvent("results processed", trace.WithAttributes(
+			attribute.Int64("state_transitions", int64(len(processedStates))),
+			attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))),
+		))
 		if len(alerts.PostableAlerts) > 0 {
 			sch.alertsSender.Send(key, alerts)
 		}
@@ -517,16 +508,17 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 						logger.Debug("Skip rule evaluation because it is paused")
 						return nil
 					}
-					tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution")
-					defer span.End()

-					span.SetAttributes("rule_uid", ctx.rule.UID, attribute.String("rule_uid", ctx.rule.UID))
-					span.SetAttributes("org_id", ctx.rule.OrgID, attribute.Int64("org_id", ctx.rule.OrgID))
-					span.SetAttributes("rule_version", ctx.rule.Version, attribute.Int64("rule_version", ctx.rule.Version))
 					fpStr := currentFingerprint.String()
-					span.SetAttributes("rule_fingerprint", fpStr, attribute.String("rule_fingerprint", fpStr))
 					utcTick := ctx.scheduledAt.UTC().Format(time.RFC3339Nano)
-					span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
+					tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution", trace.WithAttributes(
+						attribute.String("rule_uid", ctx.rule.UID),
+						attribute.Int64("org_id", ctx.rule.OrgID),
+						attribute.Int64("rule_version", ctx.rule.Version),
+						attribute.String("rule_fingerprint", fpStr),
+						attribute.String("tick", utcTick),
+					))
+					defer span.End()

 					evaluate(tracingCtx, f, attempt, ctx, span)
 					return nil
@@ -11,10 +11,10 @@ import (

 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/annotations"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -73,7 +73,7 @@ func (h *AnnotationBackend) Record(ctx context.Context, rule history_model.RuleM
 	writeCtx := context.Background()
 	writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
 	writeCtx = history_model.WithRuleData(writeCtx, rule)
-	writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
+	writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))

 	go func(ctx context.Context) {
 		defer cancel()
@@ -11,10 +11,10 @@ import (
 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/weaveworks/common/http/client"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -89,7 +89,7 @@ func (h *RemoteLokiBackend) Record(ctx context.Context, rule history_model.RuleM
 	writeCtx := context.Background()
 	writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
 	writeCtx = history_model.WithRuleData(writeCtx, rule)
-	writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
+	writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))

 	go func(ctx context.Context) {
 		defer cancel()
@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/dskit/concurrency"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -251,42 +252,33 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.Alert
 // ProcessEvalResults updates the current states that belong to a rule with the evaluation results.
 // if extraLabels is not empty, those labels will be added to every state. The extraLabels take precedence over rule labels and result labels
 func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels) []StateTransition {
-	tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation")
-	defer span.End()
-	span.SetAttributes("rule_uid", alertRule.UID, attribute.String("rule_uid", alertRule.UID))
-	span.SetAttributes("org_id", alertRule.OrgID, attribute.Int64("org_id", alertRule.OrgID))
-	span.SetAttributes("rule_version", alertRule.Version, attribute.Int64("rule_version", alertRule.Version))
 	utcTick := evaluatedAt.UTC().Format(time.RFC3339Nano)
-	span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
-	span.SetAttributes("results", len(results), attribute.Int("tick", len(results)))
+	tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation", trace.WithAttributes(
+		attribute.String("rule_uid", alertRule.UID),
+		attribute.Int64("org_id", alertRule.OrgID),
+		attribute.Int64("rule_version", alertRule.Version),
+		attribute.String("tick", utcTick),
+		attribute.Int("results", len(results))))
+	defer span.End()

 	logger := st.log.FromContext(tracingCtx)
 	logger.Debug("State manager processing evaluation results", "resultCount", len(results))
 	states := st.setNextStateForRule(tracingCtx, alertRule, results, extraLabels, logger)
-
-	span.AddEvents([]string{"message", "state_transitions"},
-		[]tracing.EventValue{
-			{Str: "results processed"},
-			{Num: int64(len(states))},
-		})
+	span.AddEvent("results processed", trace.WithAttributes(
+		attribute.Int64("state_transitions", int64(len(states))),
+	))

 	staleStates := st.deleteStaleStatesFromCache(ctx, logger, evaluatedAt, alertRule)
 	st.deleteAlertStates(tracingCtx, logger, staleStates)

 	if len(staleStates) > 0 {
-		span.AddEvents([]string{"message", "state_transitions"},
-			[]tracing.EventValue{
-				{Str: "deleted stale states"},
-				{Num: int64(len(staleStates))},
-			})
+		span.AddEvent("deleted stale states", trace.WithAttributes(
+			attribute.Int64("state_transitions", int64(len(staleStates))),
+		))
 	}

 	st.saveAlertStates(tracingCtx, logger, states...)
-
-	span.AddEvents([]string{"message"},
-		[]tracing.EventValue{
-			{Str: "updated database"},
-		})
+	span.AddEvent("updated database")

 	allChanges := append(states, staleStates...)
 	if st.historian != nil {
@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -32,10 +33,10 @@ type TracingMiddleware struct {

 // setSpanAttributeFromHTTPHeader takes a ReqContext and a span, and adds the specified HTTP header as a span attribute
 // (string value), if the header is present.
-func setSpanAttributeFromHTTPHeader(headers http.Header, span tracing.Span, attributeName, headerName string) {
+func setSpanAttributeFromHTTPHeader(headers http.Header, span trace.Span, attributeName, headerName string) {
 	// Set the attribute as string
 	if v := headers.Get(headerName); v != "" {
-		span.SetAttributes(attributeName, v, attribute.Key(attributeName).String(v))
+		span.SetAttributes(attribute.String(attributeName, v))
 	}
 }

@@ -46,23 +47,24 @@ func (m *TracingMiddleware) traceWrap(
 	ctx context.Context, pluginContext backend.PluginContext, opName string,
 ) (context.Context, func(error)) {
 	// Start span
-	ctx, span := m.tracer.Start(ctx, "PluginClient."+opName)
+	ctx, span := m.tracer.Start(ctx, "PluginClient."+opName, trace.WithAttributes(
+		// Attach some plugin context information to span
+		attribute.String("plugin_id", pluginContext.PluginID),
+		attribute.Int64("org_id", pluginContext.OrgID),
+	))

-	// Attach some plugin context information to span
-	span.SetAttributes("plugin_id", pluginContext.PluginID, attribute.String("plugin_id", pluginContext.PluginID))
-	span.SetAttributes("org_id", pluginContext.OrgID, attribute.Int64("org_id", pluginContext.OrgID))
 	if settings := pluginContext.DataSourceInstanceSettings; settings != nil {
-		span.SetAttributes("datasource_name", settings.Name, attribute.Key("datasource_name").String(settings.Name))
-		span.SetAttributes("datasource_uid", settings.UID, attribute.Key("datasource_uid").String(settings.UID))
+		span.SetAttributes(attribute.String("datasource_name", settings.Name))
+		span.SetAttributes(attribute.String("datasource_uid", settings.UID))
 	}
 	if u := pluginContext.User; u != nil {
-		span.SetAttributes("user", u.Login, attribute.String("user", u.Login))
+		span.SetAttributes(attribute.String("user", u.Login))
 	}

 	// Additional attributes from http headers
 	if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil && len(reqCtx.Req.Header) > 0 {
 		if v, err := strconv.Atoi(reqCtx.Req.Header.Get(query.HeaderPanelID)); err == nil {
-			span.SetAttributes("panel_id", v, attribute.Key("panel_id").Int(v))
+			span.SetAttributes(attribute.Int("panel_id", v))
 		}
 		setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "query_group_id", query.HeaderQueryGroupID)
 		setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "dashboard_uid", query.HeaderDashboardUID)
@@ -15,6 +15,7 @@ import (

 	"github.com/blugelabs/bluge"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"

 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -466,8 +467,9 @@ func (i *searchIndex) reportSizeOfIndexDiskBackup(orgID int64) {
 }

 func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, error) {
-	spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex")
-	span.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
+	spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex", trace.WithAttributes(
+		attribute.Int64("org_id", orgID),
+	))

 	started := time.Now()
 	ctx, cancel := context.WithTimeout(spanCtx, time.Minute)
@@ -489,9 +491,10 @@ func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, erro

 	dashboardExtender := i.extender.GetDashboardExtender(orgID)

-	_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index")
-	initOrgIndexSpan.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
-	initOrgIndexSpan.SetAttributes("dashboardCount", len(dashboards), attribute.Key("dashboardCount").Int(len(dashboards)))
+	_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index", trace.WithAttributes(
+		attribute.Int64("org_id", orgID),
+		attribute.Int("dashboardCount", len(dashboards)),
+	))

 	index, err := initOrgIndex(dashboards, i.logger, dashboardExtender)

@@ -837,10 +840,11 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
 			default:
 			}

-			dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery")
-			dashboardQuerySpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
-			dashboardQuerySpan.SetAttributes("dashboardUID", dashboardUID, attribute.Key("dashboardUID").String(dashboardUID))
-			dashboardQuerySpan.SetAttributes("lastID", lastID, attribute.Key("lastID").Int64(lastID))
+			dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery", trace.WithAttributes(
+				attribute.Int64("orgID", orgID),
+				attribute.String("dashboardUID", dashboardUID),
+				attribute.Int64("lastID", lastID),
+			))

 			rows := make([]*dashboardQueryResult, 0)
 			err := l.sql.WithDbSession(dashboardQueryCtx, func(sess *db.Session) error {
@@ -887,9 +891,9 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
 }

 func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
-	ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards")
-	span.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
-
+	ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards", trace.WithAttributes(
+		attribute.Int64("orgID", orgID),
+	))
 	defer span.End()

 	var dashboards []dashboard
@@ -901,8 +905,9 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 		dashboards = make([]dashboard, 0, limit)
 	}

-	loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup")
-	loadDatasourceSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
+	loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup", trace.WithAttributes(
+		attribute.Int64("orgID", orgID),
+	))

 	// key will allow name or uid
 	lookup, err := kdash.LoadDatasourceLookup(loadDatasourceCtx, orgID, l.sql)
@@ -930,9 +935,10 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das

 		rows := res.dashboards

-		_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard")
-		readDashboardSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
-		readDashboardSpan.SetAttributes("dashboardCount", len(rows), attribute.Key("dashboardCount").Int(len(rows)))
+		_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard", trace.WithAttributes(
+			attribute.Int64("orgID", orgID),
+			attribute.Int("dashboardCount", len(rows)),
+		))

 		reader := kdash.NewStaticDashboardSummaryBuilder(lookup, false)

@@ -13,6 +13,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/mattn/go-sqlite3"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"xorm.io/core"

@@ -100,10 +101,11 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
 	_, span := h.tracer.Start(ctx, "database query", trace.WithTimestamp(begin))
 	defer span.End()

-	span.AddEvents([]string{"query", "status"}, []tracing.EventValue{{Str: query}, {Str: status}})
+	span.AddEvent("query", trace.WithAttributes(attribute.String("query", query)))
+	span.AddEvent("status", trace.WithAttributes(attribute.String("status", status)))

 	if err != nil {
-		span.AddEvents([]string{"error"}, []tracing.EventValue{{Str: err.Error()}})
+		span.RecordError(err)
 	}

 	ctxLogger := h.log.FromContext(ctx)
@@ -9,6 +9,7 @@ import (

 	"github.com/mattn/go-sqlite3"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"xorm.io/xorm"

 	"github.com/grafana/grafana/pkg/infra/log"
@@ -37,7 +38,7 @@ func (sess *DBSession) PublishAfterCommit(msg any) {
 	sess.events = append(sess.events, msg)
 }

-func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, tracing.Span, error) {
+func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, trace.Span, error) {
 	value := ctx.Value(ContextSessionKey{})
 	var sess *DBSession
 	sess, ok := value.(*DBSession)
@@ -50,7 +51,7 @@ func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTr
 	}

 	tctx, span := tracer.Start(ctx, "open session")
-	span.SetAttributes("transaction", beginTran, attribute.Key("transaction").Bool(beginTran))
+	span.SetAttributes(attribute.Bool("transaction", beginTran))

 	newSess := &DBSession{Session: engine.NewSession(), transactionOpen: beginTran}