Chore: Implement OpenTelemtry in Grafana (#42674)

* Separate Tracer interface to TracerService and Tracer

* Fix lint

* Fix:Make it possible to start spans for both opentracing and opentelemetry in ds proxy

* Add span methods, use span interface for rest of tracing

* Fix logs in tracing

* Fix tests that are related to tracing

* Fix resourcepermissions test

* Fix some tests

* Fix more tests

* Add TracingService to wire cli runner

* Remove GlobalTracer from bus

* Renaming test function

* Remove GlobalTracer from TSDB

* Replace GlobalTracer in api

* Adjust tests to the InitializeForTests func

* Remove GlobalTracer from services

* Remove GlobalTracer

* Remove bus.NewTest

* Remove Tracer interface

* Add InitializeForBus

* Simplify tests

* Clean up tests

* Rename TracerService to Tracer

* Update pkg/middleware/request_tracing.go

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* Initialize tracer before passing it to SQLStore initialization in commands

* Remove tests for opentracing

* Set span attributes correctly, remove unnecessary trace initiliazation form test

* Add tracer instance to newSQLStore

* Fix changes due to rebase

* Add modified tracing middleware test

* Fix opentracing implementation tags

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/services/alerting/conditions/query_interval_test.go

@@ -176,7 +176,6 @@ func applyScenario(t *testing.T, timeRange string, dataSourceJsonData *simplejso
 			},
 			verifier: verifier,
 		}
-
 		_, err = condition.Eval(ctx.result, reqHandler)
 
 		require.Nil(t, err)

pkg/services/alerting/engine.go

@@ -7,13 +7,12 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/ext"
-	tlog "github.com/opentracing/opentracing-go/log"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/encryption"
@@ -40,6 +39,7 @@ type AlertEngine struct {
 	log               log.Logger
 	resultHandler     resultHandler
 	usageStatsService usagestats.Service
+	tracer            tracing.Tracer
 }
 
 // IsDisabled returns true if the alerting service is disabled for this instance.
@@ -50,7 +50,7 @@ func (e *AlertEngine) IsDisabled() bool {
 // ProvideAlertEngine returns a new AlertEngine.
 func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidator models.PluginRequestValidator,
 	dataService legacydata.RequestHandler, usageStatsService usagestats.Service, encryptionService encryption.Internal,
-	cfg *setting.Cfg) *AlertEngine {
+	cfg *setting.Cfg, tracer tracing.Tracer) *AlertEngine {
 	e := &AlertEngine{
 		Cfg:               cfg,
 		RenderService:     renderer,
@@ -58,6 +58,7 @@ func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidato
 		RequestValidator:  requestValidator,
 		DataService:       dataService,
 		usageStatsService: usageStatsService,
+		tracer:            tracer,
 	}
 	e.ticker = NewTicker(time.Now(), time.Second*0, clock.New(), 1)
 	e.execQueue = make(chan *Job, 1000)
@@ -177,9 +178,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 
 	alertCtx, cancelFn := context.WithTimeout(context.Background(), setting.AlertingEvaluationTimeout)
 	cancelChan <- cancelFn
-	span := opentracing.StartSpan("alert execution")
-	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
-
+	alertCtx, span := e.tracer.Start(alertCtx, "alert execution")
 	evalContext := NewEvalContext(alertCtx, job.Rule, e.RequestValidator)
 	evalContext.Ctx = alertCtx
 
@@ -187,32 +186,37 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 		defer func() {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
-				ext.Error.Set(span, true)
-				span.LogFields(
-					tlog.Error(fmt.Errorf("%v", err)),
-					tlog.String("message", "failed to execute alert rule. panic was recovered."),
-				)
-				span.Finish()
+				span.RecordError(fmt.Errorf("%v", err))
+				span.AddEvents(
+					[]string{"error", "message"},
+					[]tracing.EventValue{
+						{Str: fmt.Sprintf("%v", err)},
+						{Str: "failed to execute alert rule. panic was recovered."},
+					})
+				span.End()
 				close(attemptChan)
 			}
 		}()
 
 		e.evalHandler.Eval(evalContext)
 
-		span.SetTag("alertId", evalContext.Rule.ID)
-		span.SetTag("dashboardId", evalContext.Rule.DashboardID)
-		span.SetTag("firing", evalContext.Firing)
-		span.SetTag("nodatapoints", evalContext.NoDataFound)
-		span.SetTag("attemptID", attemptID)
+		span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID))
+		span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID))
+		span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing))
+		span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound))
+		span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID))
 
 		if evalContext.Error != nil {
-			ext.Error.Set(span, true)
-			span.LogFields(
-				tlog.Error(evalContext.Error),
-				tlog.String("message", "alerting execution attempt failed"),
-			)
+			span.RecordError(evalContext.Error)
+			span.AddEvents(
+				[]string{"error", "message"},
+				[]tracing.EventValue{
+					{Str: fmt.Sprintf("%v", evalContext.Error)},
+					{Str: "alerting execution attempt failed"},
+				})
+
 			if attemptID < setting.AlertingMaxAttempts {
-				span.Finish()
+				span.End()
 				e.log.Debug("Job Execution attempt triggered retry", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
 				attemptChan <- (attemptID + 1)
 				return
@@ -240,7 +244,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 			}
 		}
 
-		span.Finish()
+		span.End()
 		e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
 		close(attemptChan)
 	}()

pkg/services/alerting/engine_integration_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
 	"github.com/grafana/grafana/pkg/setting"
@@ -21,7 +22,9 @@ import (
 
 func TestEngineTimeouts(t *testing.T) {
 	usMock := &usagestats.UsageStatsMock{T: t}
-	engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg())
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
+	engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer)
 	setting.AlertingNotificationTimeout = 30 * time.Second
 	setting.AlertingMaxAttempts = 3
 	engine.resultHandler = &FakeResultHandler{}

pkg/services/alerting/engine_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
@@ -45,7 +46,9 @@ func (handler *FakeResultHandler) handle(evalContext *EvalContext) error {
 func TestEngineProcessJob(t *testing.T) {
 	bus := bus.New()
 	usMock := &usagestats.UsageStatsMock{T: t}
-	engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg())
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
+	engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer)
 	setting.AlertingEvaluationTimeout = 30 * time.Second
 	setting.AlertingNotificationTimeout = 30 * time.Second
 	setting.AlertingMaxAttempts = 3