From 7ec3489cd3f71d5c584731c68211d940f11c93c3 Mon Sep 17 00:00:00 2001
From: Ludovic Viaud <ludovic.viaud@gmail.com>
Date: Fri, 21 Jul 2023 18:01:29 +0200
Subject: [PATCH] Plugins Instrumentation: add exemplar to the plugin requests
 metrics (#72071)

* Plugins Instrumentation: add exemplar to the plugin requests metrics
---
 .../self-instrumentation/docker-compose.yaml  |  1 +
 .../instrumentation/instrumentation.go        | 20 ++++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml
index a1176cf0de2..cbd2f3f76b3 100644
--- a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml
+++ b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml
@@ -6,6 +6,7 @@
       - "host.docker.internal:host-gateway"
     command: >
       --enable-feature=remote-write-receiver
+      --enable-feature=exemplar-storage
       --config.file=/etc/prometheus/prometheus.yml
       --storage.tsdb.path=/prometheus
     volumes:
diff --git a/pkg/plugins/backendplugin/instrumentation/instrumentation.go b/pkg/plugins/backendplugin/instrumentation/instrumentation.go
index 9cf399b2faa..66d55bad2b6 100644
--- a/pkg/plugins/backendplugin/instrumentation/instrumentation.go
+++ b/pkg/plugins/backendplugin/instrumentation/instrumentation.go
@@ -76,10 +76,24 @@ func instrumentPluginRequest(ctx context.Context, cfg Cfg, pluginCtx *backend.Pl
 	}
 
 	elapsed := time.Since(start)
-	pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, string(cfg.Target)).Observe(float64(elapsed / time.Millisecond))
-	pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, string(cfg.Target)).Inc()
 
-	PluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, string(cfg.Target)).Observe(elapsed.Seconds())
+	pluginRequestDurationWithLabels := pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, string(cfg.Target))
+	pluginRequestCounterWithLabels := pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, string(cfg.Target))
+	pluginRequestDurationSecondsWithLabels := PluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, string(cfg.Target))
+
+	if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" {
+		pluginRequestDurationWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar(
+			float64(elapsed/time.Millisecond), prometheus.Labels{"traceID": traceID},
+		)
+		pluginRequestCounterWithLabels.(prometheus.ExemplarAdder).AddWithExemplar(1, prometheus.Labels{"traceID": traceID})
+		pluginRequestDurationSecondsWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar(
+			elapsed.Seconds(), prometheus.Labels{"traceID": traceID},
+		)
+	} else {
+		pluginRequestDurationWithLabels.Observe(float64(elapsed / time.Millisecond))
+		pluginRequestCounterWithLabels.Inc()
+		pluginRequestDurationSecondsWithLabels.Observe(elapsed.Seconds())
+	}
 
 	if cfg.LogDatasourceRequests {
 		logParams := []interface{}{