public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Chore: Implement OpenTelemtry in Grafana (#42674)

Browse Source 
* Separate Tracer interface to TracerService and Tracer

* Fix lint

* Fix:Make it possible to start spans for both opentracing and opentelemetry in ds proxy

* Add span methods, use span interface for rest of tracing

* Fix logs in tracing

* Fix tests that are related to tracing

* Fix resourcepermissions test

* Fix some tests

* Fix more tests

* Add TracingService to wire cli runner

* Remove GlobalTracer from bus

* Renaming test function

* Remove GlobalTracer from TSDB

* Replace GlobalTracer in api

* Adjust tests to the InitializeForTests func

* Remove GlobalTracer from services

* Remove GlobalTracer

* Remove bus.NewTest

* Remove Tracer interface

* Add InitializeForBus

* Simplify tests

* Clean up tests

* Rename TracerService to Tracer

* Update pkg/middleware/request_tracing.go

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* Initialize tracer before passing it to SQLStore initialization in commands

* Remove tests for opentracing

* Set span attributes correctly, remove unnecessary trace initiliazation form test

* Add tracer instance to newSQLStore

* Fix changes due to rebase

* Add modified tracing middleware test

* Fix opentracing implementation tags

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
This commit is contained in:
idafurjes
2022-01-20 11:10:12 +01:00
committed by GitHub
co-authored by Marcus Efraimsson
parent 5b61273497
commit 30aa24a183
62 changed files with 717 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/infra/tracing/opentelemetry_tracing.go
+62 -18
View File
@@ -2,13 +2,16 @@ package tracing
import (
"context"
"net/http"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
@@ -16,25 +19,27 @@ import (
)
type Tracer interface {
Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
Run(context.Context) error
Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
Inject(context.Context, http.Header, Span)
}
type Span interface {
End()
SetAttributes(kv ...attribute.KeyValue)
SetAttributes(key string, value interface{}, kv attribute.KeyValue)
SetName(name string)
SetStatus(code codes.Code, description string)
RecordError(err error, options ...trace.EventOption)
AddEvents(keys []string, values []EventValue)
}
var (
GlobalTracer trace.Tracer
)
type OpentelemetryTracingService struct {
type Opentelemetry struct {
enabled bool
address string
log log.Logger
tracerProvider *tracesdk.TracerProvider
tracer trace.Tracer
Cfg *setting.Cfg
}
@@ -43,7 +48,12 @@ type OpentelemetrySpan struct {
span trace.Span
}
func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error {
type EventValue struct {
Str string
Num int64
}
func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
if err != nil {
return err
@@ -57,7 +67,7 @@ func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error {
return nil
}
func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) {
func (ots *Opentelemetry) initTracerProvider() (*tracesdk.TracerProvider, error) {
// Create the Jaeger exporter
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
if err != nil {
@@ -76,7 +86,7 @@ func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerPr
return tp, nil
}
func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error {
func (ots *Opentelemetry) initOpentelemetryTracer() error {
tp, err := ots.initTracerProvider()
if err != nil {
return err
@@ -89,15 +99,18 @@ func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error {
}
ots.tracerProvider = tp
GlobalTracer = otel.GetTracerProvider().Tracer("component-main")
ots.tracer = otel.GetTracerProvider().Tracer("component-main")
return nil
}
func (ots *OpentelemetryTracingService) Run(ctx context.Context) error {
func (ots *Opentelemetry) Run(ctx context.Context) error {
<-ctx.Done()
ots.log.Info("Closing tracing")
if ots.tracerProvider == nil {
return nil
}
ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
@@ -108,16 +121,47 @@ func (ots *OpentelemetryTracingService) Run(ctx context.Context) error {
return nil
}
func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
ctx, span := GlobalTracer.Start(ctx, spanName)
oSpan := OpentelemetrySpan{span: span}
return ctx, oSpan
func (ots *Opentelemetry) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
ctx, span := ots.tracer.Start(ctx, spanName)
opentelemetrySpan := OpentelemetrySpan{
span: span,
}
return ctx, opentelemetrySpan
}
func (ots *Opentelemetry) Inject(ctx context.Context, header http.Header, _ Span) {
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(header))
}
func (s OpentelemetrySpan) End() {
s.span.End()
}
func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) {
s.span.SetAttributes(kv...)
func (s OpentelemetrySpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) {
s.span.SetAttributes(kv)
}
func (s OpentelemetrySpan) SetName(name string) {
s.span.SetName(name)
}
func (s OpentelemetrySpan) SetStatus(code codes.Code, description string) {
s.span.SetStatus(code, description)
}
func (s OpentelemetrySpan) RecordError(err error, options ...trace.EventOption) {
for _, o := range options {
s.span.RecordError(err, o)
}
}
func (s OpentelemetrySpan) AddEvents(keys []string, values []EventValue) {
for i, v := range values {
if v.Num != 0 {
s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).String(v.Str)))
}
if v.Str != "" {
s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).Int64(v.Num)))
}
}
}
pkg/infra/tracing/test_helper.go
+16
View File
@@ -0,0 +1,16 @@
package tracing
func InitializeTracerForTest() (Tracer, error) {
ots := &Opentelemetry{}
err := ots.initOpentelemetryTracer()
if err != nil {
return ots, err
}
return ots, err
}
func InitializeForBus() Tracer {
ots := &Opentelemetry{}
_ = ots.initOpentelemetryTracer()
return ots
}
pkg/infra/tracing/tracing.go
+59 -18
View File
@@ -4,17 +4,21 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"strings"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"go.opentelemetry.io/otel/attribute"
trace "go.opentelemetry.io/otel/trace"
opentracing "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
ol "github.com/opentracing/opentracing-go/log"
jaegercfg "github.com/uber/jaeger-client-go/config"
"github.com/uber/jaeger-client-go/zipkin"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
trace "go.opentelemetry.io/otel/trace"
)
const (
@@ -23,7 +27,7 @@ const (
)
func ProvideService(cfg *setting.Cfg) (Tracer, error) {
ts := &TracingService{
ts := &Opentracing{
Cfg: cfg,
log: log.New("tracing"),
}
@@ -36,7 +40,7 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
return ts, ts.initGlobalTracer()
}
ots := &OpentelemetryTracingService{
ots := &Opentelemetry{
Cfg: cfg,
log: log.New("tracing"),
}
@@ -48,7 +52,7 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
return ots, ots.initOpentelemetryTracer()
}
type TracingService struct {
type Opentracing struct {
enabled bool
address string
customTags map[string]string
@@ -67,7 +71,7 @@ type OpentracingSpan struct {
span opentracing.Span
}
func (ts *TracingService) parseSettings() error {
func (ts *Opentracing) parseSettings() error {
var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger")
if err != nil {
return err
@@ -94,7 +98,7 @@ func (ts *TracingService) parseSettings() error {
return nil
}
func (ts *TracingService) initJaegerCfg() (jaegercfg.Configuration, error) {
func (ts *Opentracing) initJaegerCfg() (jaegercfg.Configuration, error) {
cfg := jaegercfg.Configuration{
ServiceName: "grafana",
Disabled: !ts.enabled,
@@ -116,7 +120,7 @@ func (ts *TracingService) initJaegerCfg() (jaegercfg.Configuration, error) {
return cfg, nil
}
func (ts *TracingService) initGlobalTracer() error {
func (ts *Opentracing) initGlobalTracer() error {
cfg, err := ts.initJaegerCfg()
if err != nil {
return err
@@ -151,11 +155,10 @@ func (ts *TracingService) initGlobalTracer() error {
opentracing.SetGlobalTracer(tracer)
ts.closer = closer
return nil
}
func (ts *TracingService) Run(ctx context.Context) error {
func (ts *Opentracing) Run(ctx context.Context) error {
<-ctx.Done()
if ts.closer != nil {
@@ -166,22 +169,60 @@ func (ts *TracingService) Run(ctx context.Context) error {
return nil
}
func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
func (ts *Opentracing) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
oSpan := OpentracingSpan{
span: span,
opentracingSpan := OpentracingSpan{span: span}
return ctx, opentracingSpan
}
func (ts *Opentracing) Inject(ctx context.Context, header http.Header, span Span) {
opentracingSpan, ok := span.(OpentracingSpan)
if !ok {
logger.Error("Failed to cast opentracing span")
}
err := opentracing.GlobalTracer().Inject(
opentracingSpan.span.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(header))
if err != nil {
logger.Error("Failed to inject span context instance", "err", err)
}
return ctx, oSpan
}
func (s OpentracingSpan) End() {
s.span.Finish()
}
func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) {
for k, v := range kv {
s.span.SetTag(fmt.Sprint(k), v)
func (s OpentracingSpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) {
s.span.SetTag(key, value)
}
func (s OpentracingSpan) SetName(name string) {
s.span.SetOperationName(name)
}
func (s OpentracingSpan) SetStatus(code codes.Code, description string) {
ext.Error.Set(s.span, true)
}
func (s OpentracingSpan) RecordError(err error, options ...trace.EventOption) {
ext.Error.Set(s.span, true)
}
func (s OpentracingSpan) AddEvents(keys []string, values []EventValue) {
fields := []ol.Field{}
for i, v := range values {
if v.Str != "" {
field := ol.String(keys[i], v.Str)
fields = append(fields, field)
}
if v.Num != 0 {
field := ol.Int64(keys[i], v.Num)
fields = append(fields, field)
}
}
s.span.LogFields(fields...)
}
func splitTagSettings(input string) map[string]string {
pkg/infra/tracing/tracing_test.go
+7 -7
View File
@@ -42,7 +42,7 @@ func TestGroupSplit(t *testing.T) {
}
func TestInitJaegerCfg_Default(t *testing.T) {
ts := &TracingService{}
ts := &Opentracing{}
cfg, err := ts.initJaegerCfg()
require.NoError(t, err)
@@ -50,7 +50,7 @@ func TestInitJaegerCfg_Default(t *testing.T) {
}
func TestInitJaegerCfg_Enabled(t *testing.T) {
ts := &TracingService{enabled: true}
ts := &Opentracing{enabled: true}
cfg, err := ts.initJaegerCfg()
require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestInitJaegerCfg_DisabledViaEnv(t *testing.T) {
require.NoError(t, err)
}()
ts := &TracingService{enabled: true}
ts := &Opentracing{enabled: true}
cfg, err := ts.initJaegerCfg()
require.NoError(t, err)
@@ -81,7 +81,7 @@ func TestInitJaegerCfg_EnabledViaEnv(t *testing.T) {
require.NoError(t, err)
}()
ts := &TracingService{enabled: false}
ts := &Opentracing{enabled: false}
cfg, err := ts.initJaegerCfg()
require.NoError(t, err)
@@ -96,7 +96,7 @@ func TestInitJaegerCfg_InvalidEnvVar(t *testing.T) {
require.NoError(t, err)
}()
ts := &TracingService{}
ts := &Opentracing{}
_, err = ts.initJaegerCfg()
require.EqualError(t, err, "cannot parse env var JAEGER_DISABLED=totallybogus: strconv.ParseBool: parsing \"totallybogus\": invalid syntax")
}
@@ -108,7 +108,7 @@ func TestInitJaegerCfg_EnabledViaHost(t *testing.T) {
}()
cfg := setting.NewCfg()
ts := &TracingService{Cfg: cfg}
ts := &Opentracing{Cfg: cfg}
_, err := ts.Cfg.Raw.NewSection("tracing.jaeger")
require.NoError(t, err)
require.NoError(t, ts.parseSettings())
@@ -128,7 +128,7 @@ func TestInitJaegerCfg_EnabledViaHostPort(t *testing.T) {
}()
cfg := setting.NewCfg()
ts := &TracingService{Cfg: cfg}
ts := &Opentracing{Cfg: cfg}
_, err := ts.Cfg.Raw.NewSection("tracing.jaeger")
require.NoError(t, err)
require.NoError(t, ts.parseSettings())