diff --git a/conf/defaults.ini b/conf/defaults.ini index 2224e9c8fcb..6676be54b45 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -914,6 +914,10 @@ zipkin_propagation = false # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure. disable_shared_zipkin_spans = false +[tracing.opentelemetry.jaeger] +# jaeger destination (ex http://localhost:14268/api/traces) +address = + #################################### External Image Storage ############## [external_image_storage] # Used for uploading images to public servers so they can be included in slack/email messages. diff --git a/conf/sample.ini b/conf/sample.ini index 95928af34a4..9f2a590210f 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -891,6 +891,10 @@ # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure. ;disable_shared_zipkin_spans = false +[tracing.opentelemetry.jaeger] +# jaeger destination (ex http://localhost:14268/api/traces) +; address = http://localhost:14268/api/traces + #################################### External image storage ########################## [external_image_storage] # Used for uploading images to public servers so they can be included in slack/email messages. diff --git a/go.mod b/go.mod index f778cfdba7f..f7eac0ea282 100644 --- a/go.mod +++ b/go.mod @@ -101,10 +101,10 @@ require ( github.com/yudai/gojsondiff v1.0.0 go.opentelemetry.io/collector v0.31.0 go.opentelemetry.io/collector/model v0.31.0 - go.opentelemetry.io/otel v1.0.0 + go.opentelemetry.io/otel v1.2.0 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 go.opentelemetry.io/otel/sdk v1.0.0 - go.opentelemetry.io/otel/trace v1.0.0 + go.opentelemetry.io/otel/trace v1.2.0 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e golang.org/x/exp v0.0.0-20210220032938-85be41e4509f // indirect golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f @@ -133,7 +133,6 @@ require ( cloud.google.com/go v0.94.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect @@ -189,7 +188,6 @@ require ( github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect @@ -258,6 +256,11 @@ require ( gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect ) +require ( + github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect + github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect +) + // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream replace github.com/crewjam/saml => github.com/grafana/saml v0.0.0-20211007135653-aed1b2edd86b diff --git a/go.sum b/go.sum index e4b42548496..8b5cd46cca8 100644 --- a/go.sum +++ b/go.sum @@ -2470,8 +2470,9 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.21.0/go.mod h1: go.opentelemetry.io/contrib/zpages v0.0.0-20210722161726-7668016acb73/go.mod h1:NAkejuYm41lpyL43Fu1XdnCOYxN5NVV80/MJ03JQ/X8= go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I= -go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI= go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ= +go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U= go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ= go.opentelemetry.io/otel/internal/metric v0.21.0/go.mod h1:iOfAaY2YycsXfYD4kaRSbLx2LKmfpKObWBEv9QK5zFo= @@ -2481,8 +2482,9 @@ go.opentelemetry.io/otel/sdk v1.0.0-RC1/go.mod h1:kj6yPn7Pgt5ByRuwesbaWcRLA+V7BS go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y= go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM= go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg= -go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4= go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= +go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0= +go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index d3578b1926b..e9c591567fc 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -110,7 +110,7 @@ type HTTPServer struct { SecretsService secrets.Service DataSourcesService *datasources.Service cleanUpService *cleanup.CleanUpService - tracingService *tracing.TracingService + tracingService tracing.Tracer internalMetricsSvc *metrics.InternalMetricsService updateChecker *updatechecker.Service searchUsersService searchusers.Service @@ -136,7 +136,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi contextHandler *contexthandler.ContextHandler, schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG, libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service, - notificationService *notifications.NotificationService, tracingService *tracing.TracingService, + notificationService *notifications.NotificationService, tracingService tracing.Tracer, internalMetricsSvc *metrics.InternalMetricsService, quotaService *quota.QuotaService, socialService social.Service, oauthTokenService oauthtoken.OAuthTokenService, encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service, diff --git a/pkg/infra/tracing/opentelemetry_tracing.go b/pkg/infra/tracing/opentelemetry_tracing.go new file mode 100644 index 00000000000..7f93c6de9ad --- /dev/null +++ b/pkg/infra/tracing/opentelemetry_tracing.go @@ -0,0 +1,123 @@ +package tracing + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + trace "go.opentelemetry.io/otel/trace" +) + +type Tracer interface { + Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) + Run(context.Context) error +} + +type Span interface { + End() + SetAttributes(kv ...attribute.KeyValue) +} + +var ( + GlobalTracer trace.Tracer +) + +type OpentelemetryTracingService struct { + enabled bool + address string + log log.Logger + + tracerProvider *tracesdk.TracerProvider + + Cfg *setting.Cfg +} + +type OpentelemetrySpan struct { + span trace.Span +} + +func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error { + section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger") + if err != nil { + return err + } + + ots.address = section.Key("address").MustString("") + if ots.address != "" { + ots.enabled = true + } + + return nil +} + +func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) { + // Create the Jaeger exporter + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address))) + if err != nil { + return nil, err + } + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("grafana"), + attribute.String("environment", "production"), + )), + ) + + return tp, nil +} + +func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error { + tp, err := ots.initTracerProvider() + if err != nil { + return err + } + // Register our TracerProvider as the global so any imported + // instrumentation in the future will default to using it + // only if tracing is enabled + if ots.enabled { + otel.SetTracerProvider(tp) + } + + ots.tracerProvider = tp + GlobalTracer = otel.GetTracerProvider().Tracer("component-main") + + return nil +} + +func (ots *OpentelemetryTracingService) Run(ctx context.Context) error { + <-ctx.Done() + + ots.log.Info("Closing tracing") + ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + if err := ots.tracerProvider.Shutdown(ctxShutdown); err != nil { + return err + } + + return nil +} + +func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { + ctx, span := GlobalTracer.Start(ctx, spanName) + oSpan := OpentelemetrySpan{span: span} + return ctx, oSpan +} + +func (s OpentelemetrySpan) End() { + s.span.End() +} + +func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) { + s.span.SetAttributes(kv...) +} diff --git a/pkg/infra/tracing/tracing.go b/pkg/infra/tracing/tracing.go index 7fdb3efd477..df1c01efdb1 100644 --- a/pkg/infra/tracing/tracing.go +++ b/pkg/infra/tracing/tracing.go @@ -9,6 +9,8 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/setting" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" opentracing "github.com/opentracing/opentracing-go" jaegercfg "github.com/uber/jaeger-client-go/config" @@ -20,11 +22,12 @@ const ( envJaegerAgentPort = "JAEGER_AGENT_PORT" ) -func ProvideService(cfg *setting.Cfg) (*TracingService, error) { +func ProvideService(cfg *setting.Cfg) (Tracer, error) { ts := &TracingService{ Cfg: cfg, log: log.New("tracing"), } + if err := ts.parseSettings(); err != nil { return nil, err } @@ -33,7 +36,16 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) { return ts, ts.initGlobalTracer() } - return ts, nil + ots := &OpentelemetryTracingService{ + Cfg: cfg, + log: log.New("tracing"), + } + + if err := ots.parseSettingsOpentelemetry(); err != nil { + return nil, err + } + + return ots, ots.initOpentelemetryTracer() } type TracingService struct { @@ -51,6 +63,10 @@ type TracingService struct { Cfg *setting.Cfg } +type OpentracingSpan struct { + span opentracing.Span +} + func (ts *TracingService) parseSettings() error { var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger") if err != nil { @@ -150,6 +166,24 @@ func (ts *TracingService) Run(ctx context.Context) error { return nil } +func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { + span, ctx := opentracing.StartSpanFromContext(ctx, spanName) + oSpan := OpentracingSpan{ + span: span, + } + return ctx, oSpan +} + +func (s OpentracingSpan) End() { + s.span.Finish() +} + +func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) { + for k, v := range kv { + s.span.SetTag(fmt.Sprint(k), v) + } +} + func splitTagSettings(input string) map[string]string { res := map[string]string{} diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go index 1f83afd1211..453774bd0a0 100644 --- a/pkg/server/backgroundsvcs/background_services.go +++ b/pkg/server/backgroundsvcs/background_services.go @@ -45,7 +45,7 @@ func ProvideBackgroundServiceRegistry( rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService, provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, pm *manager.PluginManager, metrics *metrics.InternalMetricsService, usageStats *uss.UsageStats, updateChecker *updatechecker.Service, - tracing *tracing.TracingService, remoteCache *remotecache.RemoteCache, + tracing tracing.Tracer, remoteCache *remotecache.RemoteCache, // Need to make sure these are initialized, is there a better place to put them? _ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service, _ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,