Plugins: Improve instrumentation by adding metrics and tracing (#61035)

* WIP: Plugins tracing * Trace ID middleware * Add prometheus metrics and tracing to plugins updater * Add TODOs * Add instrumented http client * Add tracing to grafana update checker * Goimports * Moved plugins tracing to middleware * goimports, fix tests * Removed X-Trace-Id header * Fix comment in NewTracingHeaderMiddleware * Add metrics to instrumented http client * Add instrumented http client options * Removed unused function * Switch to contextual logger * Refactoring, fix tests * Moved InstrumentedHTTPClient and PrometheusMetrics to their own package * Tracing middleware: handle errors * Report span status codes when recording errors * Add tests for tracing middleware * Moved fakeSpan and fakeTracer to pkg/infra/tracing * Add TestHTTPClientTracing * Lint * Changes after PR review * Tests: Made "ended" in FakeSpan private, allow calling End only once * Testing: panic in FakeSpan if span already ended * Refactoring: Simplify Grafana updater checks * Refactoring: Simplify plugins updater error checks and logs * Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates * Tests: Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates * Log update checks duration, use Info log level for check succeeded logs * Add plugin context span attributes in tracing_middleware * Refactor prometheus metrics as httpclient middleware * Fix call to ProvidePluginsService in plugins_test.go * Propagate context to update checker outgoing http requests * Plugin client tracing middleware: Removed operation name in status * Fix tests * Goimports tracing_middleware.go * Goimports * Fix imports * Changed span name to plugins client middleware * Add span name assertion in TestTracingMiddleware * Removed Prometheus metrics middleware from grafana and plugins updatechecker * Add span attributes for ds name, type, uid, panel and dashboard ids * Fix http header reading in tracing middlewares * Use contexthandler.FromContext, add X-Query-Group-Id * Add test for RunStream * Fix imports * Changes from PR review * TestTracingMiddleware: Changed assert to require for didPanic assertion * Lint * Fix imports
2023-03-28 11:01:06 +02:00
parent 0beb768427
commit a89202eab2
16 changed files with 1028 additions and 72 deletions
@@ -1,8 +1,14 @@
 package tracing

 import (
+	"context"
+	"net/http"
+
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
 )

 func InitializeTracerForTest() Tracer {
@@ -14,3 +20,95 @@ func InitializeTracerForTest() Tracer {
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }
+
+type FakeSpan struct {
+	Name string
+
+	ended       bool
+	Attributes  map[attribute.Key]attribute.Value
+	StatusCode  codes.Code
+	Description string
+	Err         error
+	Events      map[string]EventValue
+}
+
+func newFakeSpan(name string) *FakeSpan {
+	return &FakeSpan{
+		Name:       name,
+		Attributes: map[attribute.Key]attribute.Value{},
+		Events:     map[string]EventValue{},
+	}
+}
+
+func (t *FakeSpan) End() {
+	if t.ended {
+		panic("End already called")
+	}
+	t.ended = true
+}
+
+func (t *FakeSpan) IsEnded() bool {
+	return t.ended
+}
+
+func (t *FakeSpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) {
+	if t.IsEnded() {
+		panic("span already ended")
+	}
+	t.Attributes[kv.Key] = kv.Value
+}
+
+func (t *FakeSpan) SetName(name string) {
+	if t.IsEnded() {
+		panic("span already ended")
+	}
+	t.Name = name
+}
+
+func (t *FakeSpan) SetStatus(code codes.Code, description string) {
+	if t.IsEnded() {
+		panic("span already ended")
+	}
+	t.StatusCode = code
+	t.Description = description
+}
+
+func (t *FakeSpan) RecordError(err error, options ...trace.EventOption) {
+	if t.IsEnded() {
+		panic("span already ended")
+	}
+	t.Err = err
+}
+
+func (t *FakeSpan) AddEvents(keys []string, values []EventValue) {
+	if t.IsEnded() {
+		panic("span already ended")
+	}
+	if len(keys) != len(values) {
+		panic("different number of keys and values")
+	}
+	for i := 0; i < len(keys); i++ {
+		t.Events[keys[i]] = values[i]
+	}
+}
+
+type FakeTracer struct {
+	Spans []*FakeSpan
+}
+
+func (t *FakeTracer) Run(ctx context.Context) error {
+	return nil
+}
+
+func (t *FakeTracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	span := newFakeSpan(spanName)
+	t.Spans = append(t.Spans, span)
+	return ctx, span
+}
+
+func (t *FakeTracer) Inject(ctx context.Context, header http.Header, span Span) {
+}
+
+func NewFakeTracer() *FakeTracer {
+	return &FakeTracer{Spans: []*FakeSpan{}}
+}