diff --git a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml index a1176cf0de2..cbd2f3f76b3 100644 --- a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml +++ b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml @@ -6,6 +6,7 @@ - "host.docker.internal:host-gateway" command: > --enable-feature=remote-write-receiver + --enable-feature=exemplar-storage --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus volumes: diff --git a/pkg/plugins/backendplugin/instrumentation/instrumentation.go b/pkg/plugins/backendplugin/instrumentation/instrumentation.go index 9cf399b2faa..66d55bad2b6 100644 --- a/pkg/plugins/backendplugin/instrumentation/instrumentation.go +++ b/pkg/plugins/backendplugin/instrumentation/instrumentation.go @@ -76,10 +76,24 @@ func instrumentPluginRequest(ctx context.Context, cfg Cfg, pluginCtx *backend.Pl } elapsed := time.Since(start) - pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, string(cfg.Target)).Observe(float64(elapsed / time.Millisecond)) - pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, string(cfg.Target)).Inc() - PluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, string(cfg.Target)).Observe(elapsed.Seconds()) + pluginRequestDurationWithLabels := pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, string(cfg.Target)) + pluginRequestCounterWithLabels := pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, string(cfg.Target)) + pluginRequestDurationSecondsWithLabels := PluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, string(cfg.Target)) + + if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" { + pluginRequestDurationWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( + float64(elapsed/time.Millisecond), prometheus.Labels{"traceID": traceID}, + ) + pluginRequestCounterWithLabels.(prometheus.ExemplarAdder).AddWithExemplar(1, prometheus.Labels{"traceID": traceID}) + pluginRequestDurationSecondsWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( + elapsed.Seconds(), prometheus.Labels{"traceID": traceID}, + ) + } else { + pluginRequestDurationWithLabels.Observe(float64(elapsed / time.Millisecond)) + pluginRequestCounterWithLabels.Inc() + pluginRequestDurationSecondsWithLabels.Observe(elapsed.Seconds()) + } if cfg.LogDatasourceRequests { logParams := []interface{}{