Chore: Implement OpenTelemtry in Grafana (#42674)

* Separate Tracer interface to TracerService and Tracer

* Fix lint

* Fix:Make it possible to start spans for both opentracing and opentelemetry in ds proxy

* Add span methods, use span interface for rest of tracing

* Fix logs in tracing

* Fix tests that are related to tracing

* Fix resourcepermissions test

* Fix some tests

* Fix more tests

* Add TracingService to wire cli runner

* Remove GlobalTracer from bus

* Renaming test function

* Remove GlobalTracer from TSDB

* Replace GlobalTracer in api

* Adjust tests to the InitializeForTests func

* Remove GlobalTracer from services

* Remove GlobalTracer

* Remove bus.NewTest

* Remove Tracer interface

* Add InitializeForBus

* Simplify tests

* Clean up tests

* Rename TracerService to Tracer

* Update pkg/middleware/request_tracing.go

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* Initialize tracer before passing it to SQLStore initialization in commands

* Remove tests for opentracing

* Set span attributes correctly, remove unnecessary trace initiliazation form test

* Add tracer instance to newSQLStore

* Fix changes due to rebase

* Add modified tracing middleware test

* Fix opentracing implementation tags

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/services/alerting/conditions/query_interval_test.go

@@ -176,7 +176,6 @@ func applyScenario(t *testing.T, timeRange string, dataSourceJsonData *simplejso
 			},
 			verifier: verifier,
 		}
-
 		_, err = condition.Eval(ctx.result, reqHandler)
 
 		require.Nil(t, err)

pkg/services/alerting/engine.go

@@ -7,13 +7,12 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/ext"
-	tlog "github.com/opentracing/opentracing-go/log"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/encryption"
@@ -40,6 +39,7 @@ type AlertEngine struct {
 	log               log.Logger
 	resultHandler     resultHandler
 	usageStatsService usagestats.Service
+	tracer            tracing.Tracer
 }
 
 // IsDisabled returns true if the alerting service is disabled for this instance.
@@ -50,7 +50,7 @@ func (e *AlertEngine) IsDisabled() bool {
 // ProvideAlertEngine returns a new AlertEngine.
 func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidator models.PluginRequestValidator,
 	dataService legacydata.RequestHandler, usageStatsService usagestats.Service, encryptionService encryption.Internal,
-	cfg *setting.Cfg) *AlertEngine {
+	cfg *setting.Cfg, tracer tracing.Tracer) *AlertEngine {
 	e := &AlertEngine{
 		Cfg:               cfg,
 		RenderService:     renderer,
@@ -58,6 +58,7 @@ func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidato
 		RequestValidator:  requestValidator,
 		DataService:       dataService,
 		usageStatsService: usageStatsService,
+		tracer:            tracer,
 	}
 	e.ticker = NewTicker(time.Now(), time.Second*0, clock.New(), 1)
 	e.execQueue = make(chan *Job, 1000)
@@ -177,9 +178,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 
 	alertCtx, cancelFn := context.WithTimeout(context.Background(), setting.AlertingEvaluationTimeout)
 	cancelChan <- cancelFn
-	span := opentracing.StartSpan("alert execution")
-	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
-
+	alertCtx, span := e.tracer.Start(alertCtx, "alert execution")
 	evalContext := NewEvalContext(alertCtx, job.Rule, e.RequestValidator)
 	evalContext.Ctx = alertCtx
 
@@ -187,32 +186,37 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 		defer func() {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
-				ext.Error.Set(span, true)
-				span.LogFields(
-					tlog.Error(fmt.Errorf("%v", err)),
-					tlog.String("message", "failed to execute alert rule. panic was recovered."),
-				)
-				span.Finish()
+				span.RecordError(fmt.Errorf("%v", err))
+				span.AddEvents(
+					[]string{"error", "message"},
+					[]tracing.EventValue{
+						{Str: fmt.Sprintf("%v", err)},
+						{Str: "failed to execute alert rule. panic was recovered."},
+					})
+				span.End()
 				close(attemptChan)
 			}
 		}()
 
 		e.evalHandler.Eval(evalContext)
 
-		span.SetTag("alertId", evalContext.Rule.ID)
-		span.SetTag("dashboardId", evalContext.Rule.DashboardID)
-		span.SetTag("firing", evalContext.Firing)
-		span.SetTag("nodatapoints", evalContext.NoDataFound)
-		span.SetTag("attemptID", attemptID)
+		span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID))
+		span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID))
+		span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing))
+		span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound))
+		span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID))
 
 		if evalContext.Error != nil {
-			ext.Error.Set(span, true)
-			span.LogFields(
-				tlog.Error(evalContext.Error),
-				tlog.String("message", "alerting execution attempt failed"),
-			)
+			span.RecordError(evalContext.Error)
+			span.AddEvents(
+				[]string{"error", "message"},
+				[]tracing.EventValue{
+					{Str: fmt.Sprintf("%v", evalContext.Error)},
+					{Str: "alerting execution attempt failed"},
+				})
+
 			if attemptID < setting.AlertingMaxAttempts {
-				span.Finish()
+				span.End()
 				e.log.Debug("Job Execution attempt triggered retry", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
 				attemptChan <- (attemptID + 1)
 				return
@@ -240,7 +244,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 			}
 		}
 
-		span.Finish()
+		span.End()
 		e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
 		close(attemptChan)
 	}()

pkg/services/alerting/engine_integration_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
 	"github.com/grafana/grafana/pkg/setting"
@@ -21,7 +22,9 @@ import (
 
 func TestEngineTimeouts(t *testing.T) {
 	usMock := &usagestats.UsageStatsMock{T: t}
-	engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg())
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
+	engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer)
 	setting.AlertingNotificationTimeout = 30 * time.Second
 	setting.AlertingMaxAttempts = 3
 	engine.resultHandler = &FakeResultHandler{}

pkg/services/alerting/engine_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
@@ -45,7 +46,9 @@ func (handler *FakeResultHandler) handle(evalContext *EvalContext) error {
 func TestEngineProcessJob(t *testing.T) {
 	bus := bus.New()
 	usMock := &usagestats.UsageStatsMock{T: t}
-	engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg())
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
+	engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer)
 	setting.AlertingEvaluationTimeout = 30 * time.Second
 	setting.AlertingNotificationTimeout = 30 * time.Second
 	setting.AlertingMaxAttempts = 3

pkg/services/contexthandler/auth_proxy_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/remotecache"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/auth"
 	"github.com/grafana/grafana/pkg/services/contexthandler/authproxy"
@@ -102,6 +103,8 @@ func getContextHandler(t *testing.T) *ContextHandler {
 	userAuthTokenSvc := auth.NewFakeUserAuthTokenService()
 	renderSvc := &fakeRenderService{}
 	authJWTSvc := models.NewFakeJWTService()
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
 
-	return ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore)
+	return ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore, tracer)
 }

pkg/services/contexthandler/contexthandler.go

@@ -14,6 +14,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/network"
 	"github.com/grafana/grafana/pkg/infra/remotecache"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/middleware/cookies"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/contexthandler/authproxy"
@@ -23,8 +24,6 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util"
 	"github.com/grafana/grafana/pkg/web"
-	"github.com/opentracing/opentracing-go"
-	ol "github.com/opentracing/opentracing-go/log"
 	cw "github.com/weaveworks/common/tracing"
 )
 
@@ -36,7 +35,8 @@ const (
 const ServiceName = "ContextHandler"
 
 func ProvideService(cfg *setting.Cfg, tokenService models.UserTokenService, jwtService models.JWTService,
-	remoteCache *remotecache.RemoteCache, renderService rendering.Service, sqlStore *sqlstore.SQLStore) *ContextHandler {
+	remoteCache *remotecache.RemoteCache, renderService rendering.Service, sqlStore *sqlstore.SQLStore,
+	tracer tracing.Tracer) *ContextHandler {
 	return &ContextHandler{
 		Cfg:              cfg,
 		AuthTokenService: tokenService,
@@ -44,6 +44,7 @@ func ProvideService(cfg *setting.Cfg, tokenService models.UserTokenService, jwtS
 		RemoteCache:      remoteCache,
 		RenderService:    renderService,
 		SQLStore:         sqlStore,
+		tracer:           tracer,
 	}
 }
 
@@ -55,7 +56,7 @@ type ContextHandler struct {
 	RemoteCache      *remotecache.RemoteCache
 	RenderService    rendering.Service
 	SQLStore         *sqlstore.SQLStore
-
+	tracer           tracing.Tracer
 	// GetTime returns the current time.
 	// Stubbable by tests.
 	GetTime func() time.Time
@@ -73,8 +74,8 @@ func FromContext(c context.Context) *models.ReqContext {
 
 // Middleware provides a middleware to initialize the Macaron context.
 func (h *ContextHandler) Middleware(mContext *web.Context) {
-	span, _ := opentracing.StartSpanFromContext(mContext.Req.Context(), "Auth - Middleware")
-	defer span.Finish()
+	_, span := h.tracer.Start(mContext.Req.Context(), "Auth - Middleware")
+	defer span.End()
 
 	reqContext := &models.ReqContext{
 		Context:        mContext,
@@ -135,11 +136,13 @@ func (h *ContextHandler) Middleware(mContext *web.Context) {
 	}
 
 	reqContext.Logger = log.New("context", "userId", reqContext.UserId, "orgId", reqContext.OrgId, "uname", reqContext.Login)
-
-	span.LogFields(
-		ol.String("uname", reqContext.Login),
-		ol.Int64("orgId", reqContext.OrgId),
-		ol.Int64("userId", reqContext.UserId))
+	span.AddEvents(
+		[]string{"uname", "orgId", "userId"},
+		[]tracing.EventValue{
+			{Str: reqContext.Login},
+			{Num: reqContext.OrgId},
+			{Num: reqContext.UserId}},
+	)
 
 	mContext.Map(reqContext)
 
@@ -157,8 +160,8 @@ func (h *ContextHandler) initContextWithAnonymousUser(reqContext *models.ReqCont
 		return false
 	}
 
-	span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAnonymousUser")
-	defer span.Finish()
+	_, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAnonymousUser")
+	defer span.End()
 
 	org, err := h.SQLStore.GetOrgByName(h.Cfg.AnonymousOrgName)
 	if err != nil {
@@ -192,8 +195,8 @@ func (h *ContextHandler) initContextWithAPIKey(reqContext *models.ReqContext) bo
 		return false
 	}
 
-	span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAPIKey")
-	defer span.Finish()
+	_, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAPIKey")
+	defer span.End()
 
 	// base64 decode key
 	decoded, err := apikeygen.Decode(keyString)
@@ -272,8 +275,8 @@ func (h *ContextHandler) initContextWithBasicAuth(reqContext *models.ReqContext,
 		return false
 	}
 
-	span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithBasicAuth")
-	defer span.Finish()
+	ctx, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithBasicAuth")
+	defer span.End()
 
 	username, password, err := util.DecodeBasicAuthHeader(header)
 	if err != nil {
@@ -328,8 +331,8 @@ func (h *ContextHandler) initContextWithToken(reqContext *models.ReqContext, org
 		return false
 	}
 
-	span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithToken")
-	defer span.Finish()
+	ctx, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithToken")
+	defer span.End()
 
 	token, err := h.AuthTokenService.LookupToken(ctx, rawToken)
 	if err != nil {
@@ -369,8 +372,8 @@ func (h *ContextHandler) rotateEndOfRequestFunc(reqContext *models.ReqContext, a
 			return
 		}
 
-		span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "rotateEndOfRequestFunc")
-		defer span.Finish()
+		ctx, span := h.tracer.Start(reqContext.Req.Context(), "rotateEndOfRequestFunc")
+		defer span.End()
 
 		addr := reqContext.RemoteAddr()
 		ip, err := network.GetIPFromAddress(addr)
@@ -396,8 +399,8 @@ func (h *ContextHandler) initContextWithRenderAuth(reqContext *models.ReqContext
 		return false
 	}
 
-	span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithRenderAuth")
-	defer span.Finish()
+	_, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithRenderAuth")
+	defer span.End()
 
 	renderUser, exists := h.RenderService.GetRenderUser(reqContext.Req.Context(), key)
 	if !exists {
@@ -466,8 +469,8 @@ func (h *ContextHandler) initContextWithAuthProxy(reqContext *models.ReqContext,
 		return false
 	}
 
-	span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAuthProxy")
-	defer span.Finish()
+	_, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAuthProxy")
+	defer span.End()
 
 	// Check if allowed to continue with this IP
 	if err := auth.IsAllowedIP(); err != nil {

pkg/services/datasourceproxy/datasourceproxy.go

@@ -11,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/datasources"
@@ -21,7 +22,7 @@ import (
 
 func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator models.PluginRequestValidator,
 	pluginStore plugins.Store, cfg *setting.Cfg, httpClientProvider httpclient.Provider,
-	oauthTokenService *oauthtoken.Service, dsService *datasources.Service) *DataSourceProxyService {
+	oauthTokenService *oauthtoken.Service, dsService *datasources.Service, tracer tracing.Tracer) *DataSourceProxyService {
 	return &DataSourceProxyService{
 		DataSourceCache:        dataSourceCache,
 		PluginRequestValidator: plugReqValidator,
@@ -30,6 +31,7 @@ func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator m
 		HTTPClientProvider:     httpClientProvider,
 		OAuthTokenService:      oauthTokenService,
 		DataSourcesService:     dsService,
+		tracer:                 tracer,
 	}
 }
 
@@ -41,6 +43,7 @@ type DataSourceProxyService struct {
 	HTTPClientProvider     httpclient.Provider
 	OAuthTokenService      *oauthtoken.Service
 	DataSourcesService     *datasources.Service
+	tracer                 tracing.Tracer
 }
 
 func (p *DataSourceProxyService) ProxyDataSourceRequest(c *models.ReqContext) {
@@ -84,7 +87,7 @@ func (p *DataSourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte
 
 	proxyPath := getProxyPath(c)
 	proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider,
-		p.OAuthTokenService, p.DataSourcesService)
+		p.OAuthTokenService, p.DataSourcesService, p.tracer)
 	if err != nil {
 		if errors.Is(err, datasource.URLValidationError{}) {
 			c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

pkg/services/libraryelements/libraryelements_test.go

@@ -316,6 +316,7 @@ func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioCo
 			Name:  "User In DB",
 			Login: userInDbName,
 		}
+
 		_, err := sqlStore.CreateUser(context.Background(), cmd)
 		require.NoError(t, err)
 

pkg/services/librarypanels/librarypanels_test.go

@@ -1547,6 +1547,7 @@ func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioCo
 			Name:  "User In DB",
 			Login: userInDbName,
 		}
+
 		_, err := sqlStore.CreateUser(context.Background(), cmd)
 		require.NoError(t, err)
 

pkg/services/provisioning/plugins/plugin_provisioner_test.go

@@ -59,6 +59,7 @@ func TestPluginProvisioner(t *testing.T) {
 		}
 		reader := &testConfigReader{result: cfg}
 		ap := PluginProvisioner{log: log.New("test"), cfgProvider: reader}
+
 		err := ap.applyChanges(context.Background(), "")
 		require.NoError(t, err)
 		require.Len(t, sentCommands, 4)

pkg/services/provisioning/utils/utils_test.go

@@ -15,6 +15,7 @@ func TestCheckOrgExists(t *testing.T) {
 		sqlstore.InitTestDB(t)
 
 		defaultOrg := models.CreateOrgCommand{Name: "Main Org."}
+
 		err := sqlstore.CreateOrg(context.Background(), &defaultOrg)
 		require.NoError(t, err)
 

pkg/services/sqlstore/database_wrapper.go

@@ -11,11 +11,10 @@ import (
 	"github.com/gchaincl/sqlhooks"
 	"github.com/go-sql-driver/mysql"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
 	"github.com/lib/pq"
 	"github.com/mattn/go-sqlite3"
-	"github.com/opentracing/opentracing-go"
-	ol "github.com/opentracing/opentracing-go/log"
 	"github.com/prometheus/client_golang/prometheus"
 	cw "github.com/weaveworks/common/tracing"
 	"xorm.io/core"
@@ -39,7 +38,7 @@ func init() {
 // WrapDatabaseDriverWithHooks creates a fake database driver that
 // executes pre and post functions which we use to gather metrics about
 // database queries. It also registers the metrics.
-func WrapDatabaseDriverWithHooks(dbType string) string {
+func WrapDatabaseDriverWithHooks(dbType string, tracer tracing.Tracer) string {
 	drivers := map[string]driver.Driver{
 		migrator.SQLite:   &sqlite3.SQLiteDriver{},
 		migrator.MySQL:    &mysql.MySQLDriver{},
@@ -52,7 +51,7 @@ func WrapDatabaseDriverWithHooks(dbType string) string {
 	}
 
 	driverWithHooks := dbType + "WithHooks"
-	sql.Register(driverWithHooks, sqlhooks.Wrap(d, &databaseQueryWrapper{log: log.New("sqlstore.metrics")}))
+	sql.Register(driverWithHooks, sqlhooks.Wrap(d, &databaseQueryWrapper{log: log.New("sqlstore.metrics"), tracer: tracer}))
 	core.RegisterDriver(driverWithHooks, &databaseQueryWrapperDriver{dbType: dbType})
 	return driverWithHooks
 }
@@ -60,7 +59,8 @@ func WrapDatabaseDriverWithHooks(dbType string) string {
 // databaseQueryWrapper satisfies the sqlhook.databaseQueryWrapper interface
 // which allow us to wrap all SQL queries with a `Before` & `After` hook.
 type databaseQueryWrapper struct {
-	log log.Logger
+	log    log.Logger
+	tracer tracing.Tracer
 }
 
 // databaseQueryWrapperKey is used as key to save values in `context.Context`
@@ -94,15 +94,13 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
 		histogram.Observe(elapsed.Seconds())
 	}
 
-	span, _ := opentracing.StartSpanFromContext(ctx, "database query")
-	defer span.Finish()
+	_, span := h.tracer.Start(ctx, "database query")
+	defer span.End()
 
-	span.LogFields(
-		ol.String("query", query),
-		ol.String("status", status))
+	span.AddEvents([]string{"query", "status"}, []tracing.EventValue{{Str: query}, {Str: status}})
 
 	if err != nil {
-		span.LogFields(ol.String("error", err.Error()))
+		span.AddEvents([]string{"error"}, []tracing.EventValue{{Str: err.Error()}})
 	}
 
 	h.log.Debug("query finished", "status", status, "elapsed time", elapsed, "sql", query, "error", err)

pkg/services/sqlstore/sqlstore.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/fs"
 	"github.com/grafana/grafana/pkg/infra/localcache"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/services/annotations"
@@ -51,14 +52,16 @@ type SQLStore struct {
 	Dialect                     migrator.Dialect
 	skipEnsureDefaultOrgAndUser bool
 	migrations                  registry.DatabaseMigrator
+	tracer                      tracing.Tracer
 }
 
-func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, migrations registry.DatabaseMigrator) (*SQLStore, error) {
+func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, migrations registry.DatabaseMigrator, tracer tracing.Tracer,
+) (*SQLStore, error) {
 	// This change will make xorm use an empty default schema for postgres and
 	// by that mimic the functionality of how it was functioning before
 	// xorm's changes above.
 	xorm.DefaultPostgresSchema = ""
-	s, err := newSQLStore(cfg, cacheService, bus, nil, migrations)
+	s, err := newSQLStore(cfg, cacheService, bus, nil, migrations, tracer)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +73,7 @@ func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus
 	if err := s.Reset(); err != nil {
 		return nil, err
 	}
-
+	s.tracer = tracer
 	return s, nil
 }
 
@@ -79,7 +82,7 @@ func ProvideServiceForTests(migrations registry.DatabaseMigrator) (*SQLStore, er
 }
 
 func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, engine *xorm.Engine,
-	migrations registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQLStore, error) {
+	migrations registry.DatabaseMigrator, tracer tracing.Tracer, opts ...InitTestDBOpt) (*SQLStore, error) {
 	ss := &SQLStore{
 		Cfg:                         cfg,
 		Bus:                         bus,
@@ -87,6 +90,7 @@ func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bu
 		log:                         log.New("sqlstore"),
 		skipEnsureDefaultOrgAndUser: false,
 		migrations:                  migrations,
+		tracer:                      tracer,
 	}
 	for _, opt := range opts {
 		if !opt.EnsureDefaultOrgAndUser {
@@ -323,7 +327,7 @@ func (ss *SQLStore) initEngine(engine *xorm.Engine) error {
 	}
 
 	if ss.Cfg.IsDatabaseMetricsEnabled() {
-		ss.dbCfg.Type = WrapDatabaseDriverWithHooks(ss.dbCfg.Type)
+		ss.dbCfg.Type = WrapDatabaseDriverWithHooks(ss.dbCfg.Type, ss.tracer)
 	}
 
 	sqlog.Info("Connecting to DB", "dbtype", ss.dbCfg.Type)
@@ -528,7 +532,11 @@ func initTestDB(migration registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQ
 		engine.DatabaseTZ = time.UTC
 		engine.TZLocation = time.UTC
 
-		testSQLStore, err = newSQLStore(cfg, localcache.New(5*time.Minute, 10*time.Minute), bus.GetBus(), engine, migration, opts...)
+		tracer, err := tracing.InitializeTracerForTest()
+		if err != nil {
+			return nil, err
+		}
+		testSQLStore, err = newSQLStore(cfg, localcache.New(5*time.Minute, 10*time.Minute), bus.GetBus(), engine, migration, tracer, opts...)
 		if err != nil {
 			return nil, err
 		}

pkg/services/teamguardian/manager/service_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/teamguardian/database"
 	"github.com/stretchr/testify/mock"
@@ -32,9 +33,11 @@ func TestUpdateTeam(t *testing.T) {
 
 		t.Run("Given an editor and a team he isn't a member of", func(t *testing.T) {
 			t.Run("Should not be able to update the team", func(t *testing.T) {
+				_, err := tracing.InitializeTracerForTest()
+				require.NoError(t, err)
 				ctx := context.Background()
 				store.On("GetTeamMembers", ctx, mock.Anything).Return([]*models.TeamMemberDTO{}, nil).Once()
-				err := teamGuardianService.CanAdmin(ctx, testTeam.OrgId, testTeam.Id, &editor)
+				err = teamGuardianService.CanAdmin(ctx, testTeam.OrgId, testTeam.Id, &editor)
 				require.Equal(t, models.ErrNotAllowedToUpdateTeam, err)
 			})
 		})