chore(tracing): add tracing for frontend and db session (#91509)

This PR adds instrumentation for loading frontend SPA along with select methods in the dashboard service, and cleans up span handling in sqlstore.

---------

Co-authored-by: Dave Henderson <dave.henderson@grafana.com>

pkg/services/search/service.go

@@ -12,8 +12,11 @@ import (
 	"github.com/grafana/grafana/pkg/services/star"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
+	"go.opentelemetry.io/otel"
 )
 
+var tracer = otel.Tracer("github.com/grafana/grafana/pkg/services/search")
+
 func ProvideService(cfg *setting.Cfg, sqlstore db.DB, starService star.Service, dashboardService dashboards.DashboardService) *SearchService {
 	s := &SearchService{
 		Cfg: cfg,
@@ -61,6 +64,9 @@ type SearchService struct {
 }
 
 func (s *SearchService) SearchHandler(ctx context.Context, query *Query) (model.HitList, error) {
+	ctx, span := tracer.Start(ctx, "search.SearchHandler")
+	defer span.End()
+
 	starredQuery := star.GetUserStarsQuery{
 		UserID: query.SignedInUser.UserID,
 	}

pkg/services/sqlstore/database_wrapper.go

@@ -119,6 +119,7 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
 
 	ctx = log.IncDBCallCounter(ctx)
 
+	// timestamp overridden and recorded AFTER query is run
 	_, span := h.tracer.Start(ctx, "database query", trace.WithTimestamp(begin))
 	defer span.End()
 

pkg/services/sqlstore/session.go

@@ -10,6 +10,7 @@ import (
 	"github.com/mattn/go-sqlite3"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
 	"xorm.io/xorm"
 
 	"github.com/grafana/grafana/pkg/apimachinery/errutil"
@@ -47,10 +48,15 @@ func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTr
 		ctxLogger := sessionLogger.FromContext(ctx)
 		ctxLogger.Debug("reusing existing session", "transaction", sess.transactionOpen)
 		sess.Session = sess.Session.Context(ctx)
-		return sess, false, nil, nil
+
+		// This is a noop span to simplify later operations. purposefully not using existing context
+		_, span := noop.NewTracerProvider().Tracer("integrationtests").Start(ctx, "sqlstore.startSessionOrUseExisting")
+
+		return sess, false, span, nil
 	}
 
 	tctx, span := tracer.Start(ctx, "open session")
+
 	span.SetAttributes(attribute.Bool("transaction", beginTran))
 
 	newSess := &DBSession{Session: engine.NewSession(), transactionOpen: beginTran}
@@ -103,16 +109,14 @@ func (ss *SQLStore) retryOnLocks(ctx context.Context, callback DBTransactionFunc
 
 func (ss *SQLStore) withDbSession(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc) error {
 	sess, isNew, span, err := startSessionOrUseExisting(ctx, engine, false, ss.tracer)
+	defer span.End()
+
 	if err != nil {
-		return err
+		return tracing.Errorf(span, "start session failed: %s", err)
 	}
+
 	if isNew {
-		defer func() {
-			if span != nil {
-				span.End()
-			}
-			sess.Close()
-		}()
+		defer sess.Close()
 	}
 	retry := 0
 	return retryer.Retry(ss.retryOnLocks(ctx, callback, sess, retry), ss.dbCfg.QueryRetries, time.Millisecond*time.Duration(10), time.Second)