Add interface Tracer, add Opentelemetry (#41963)

* Add interface Tracer, add Opentelemetry

* Fix lint

* Fix failing tests and return error if config not parsed fo opentelemetry

* Update defaults.ini

Add comment with jaeger url

* go mod tidy

* Remove comments that are not needed

* Move OpentracingSpan to tracing.go

* Add opentelemetry to sample.ini

pkg/api/http_server.go

@@ -110,7 +110,7 @@ type HTTPServer struct {
 	SecretsService            secrets.Service
 	DataSourcesService        *datasources.Service
 	cleanUpService            *cleanup.CleanUpService
-	tracingService            *tracing.TracingService
+	tracingService            tracing.Tracer
 	internalMetricsSvc        *metrics.InternalMetricsService
 	updateChecker             *updatechecker.Service
 	searchUsersService        searchusers.Service
@@ -136,7 +136,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	contextHandler *contexthandler.ContextHandler,
 	schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
 	libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
-	notificationService *notifications.NotificationService, tracingService *tracing.TracingService,
+	notificationService *notifications.NotificationService, tracingService tracing.Tracer,
 	internalMetricsSvc *metrics.InternalMetricsService, quotaService *quota.QuotaService,
 	socialService social.Service, oauthTokenService oauthtoken.OAuthTokenService,
 	encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service,

pkg/infra/tracing/opentelemetry_tracing.go

@@ -0,0 +1,123 @@
+package tracing
+
+import (
+	"context"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/setting"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	trace "go.opentelemetry.io/otel/trace"
+)
+
+type Tracer interface {
+	Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
+	Run(context.Context) error
+}
+
+type Span interface {
+	End()
+	SetAttributes(kv ...attribute.KeyValue)
+}
+
+var (
+	GlobalTracer trace.Tracer
+)
+
+type OpentelemetryTracingService struct {
+	enabled bool
+	address string
+	log     log.Logger
+
+	tracerProvider *tracesdk.TracerProvider
+
+	Cfg *setting.Cfg
+}
+
+type OpentelemetrySpan struct {
+	span trace.Span
+}
+
+func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error {
+	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
+	if err != nil {
+		return err
+	}
+
+	ots.address = section.Key("address").MustString("")
+	if ots.address != "" {
+		ots.enabled = true
+	}
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) {
+	// Create the Jaeger exporter
+	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
+	if err != nil {
+		return nil, err
+	}
+
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		tracesdk.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String("grafana"),
+			attribute.String("environment", "production"),
+		)),
+	)
+
+	return tp, nil
+}
+
+func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error {
+	tp, err := ots.initTracerProvider()
+	if err != nil {
+		return err
+	}
+	// Register our TracerProvider as the global so any imported
+	// instrumentation in the future will default to using it
+	// only if tracing is enabled
+	if ots.enabled {
+		otel.SetTracerProvider(tp)
+	}
+
+	ots.tracerProvider = tp
+	GlobalTracer = otel.GetTracerProvider().Tracer("component-main")
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) Run(ctx context.Context) error {
+	<-ctx.Done()
+
+	ots.log.Info("Closing tracing")
+	ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5)
+	defer cancel()
+
+	if err := ots.tracerProvider.Shutdown(ctxShutdown); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	ctx, span := GlobalTracer.Start(ctx, spanName)
+	oSpan := OpentelemetrySpan{span: span}
+	return ctx, oSpan
+}
+
+func (s OpentelemetrySpan) End() {
+	s.span.End()
+}
+
+func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) {
+	s.span.SetAttributes(kv...)
+}

pkg/infra/tracing/tracing.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/setting"
+	"go.opentelemetry.io/otel/attribute"
+	trace "go.opentelemetry.io/otel/trace"
 
 	opentracing "github.com/opentracing/opentracing-go"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
@@ -20,11 +22,12 @@ const (
 	envJaegerAgentPort = "JAEGER_AGENT_PORT"
 )
 
-func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
+func ProvideService(cfg *setting.Cfg) (Tracer, error) {
 	ts := &TracingService{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
+
 	if err := ts.parseSettings(); err != nil {
 		return nil, err
 	}
@@ -33,7 +36,16 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
 		return ts, ts.initGlobalTracer()
 	}
 
-	return ts, nil
+	ots := &OpentelemetryTracingService{
+		Cfg: cfg,
+		log: log.New("tracing"),
+	}
+
+	if err := ots.parseSettingsOpentelemetry(); err != nil {
+		return nil, err
+	}
+
+	return ots, ots.initOpentelemetryTracer()
 }
 
 type TracingService struct {
@@ -51,6 +63,10 @@ type TracingService struct {
 	Cfg *setting.Cfg
 }
 
+type OpentracingSpan struct {
+	span opentracing.Span
+}
+
 func (ts *TracingService) parseSettings() error {
 	var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger")
 	if err != nil {
@@ -150,6 +166,24 @@ func (ts *TracingService) Run(ctx context.Context) error {
 	return nil
 }
 
+func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
+	oSpan := OpentracingSpan{
+		span: span,
+	}
+	return ctx, oSpan
+}
+
+func (s OpentracingSpan) End() {
+	s.span.Finish()
+}
+
+func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) {
+	for k, v := range kv {
+		s.span.SetTag(fmt.Sprint(k), v)
+	}
+}
+
 func splitTagSettings(input string) map[string]string {
 	res := map[string]string{}
 

pkg/server/backgroundsvcs/background_services.go

@@ -45,7 +45,7 @@ func ProvideBackgroundServiceRegistry(
 	rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService,
 	provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, pm *manager.PluginManager,
 	metrics *metrics.InternalMetricsService, usageStats *uss.UsageStats, updateChecker *updatechecker.Service,
-	tracing *tracing.TracingService, remoteCache *remotecache.RemoteCache,
+	tracing tracing.Tracer, remoteCache *remotecache.RemoteCache,
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
 	_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,