From c40bc0665b833e0a44910368535663ecffa44736 Mon Sep 17 00:00:00 2001
From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:38:14 +0200
Subject: [PATCH] Elasticsearch: Add instrumentation for
 `elasticsearch_plugin_parse_response_duration_seconds` (#75628)

* Elasticsearch: Add instrumentation for elasticsearch_plugin_parse_response_duration_seconds

* Update error tracking
---
 .../instrumentation/instrumentation.go        | 33 +++++++++++++++++++
 pkg/tsdb/elasticsearch/response_parser.go     |  7 ++--
 2 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 pkg/tsdb/elasticsearch/instrumentation/instrumentation.go

diff --git a/pkg/tsdb/elasticsearch/instrumentation/instrumentation.go b/pkg/tsdb/elasticsearch/instrumentation/instrumentation.go
new file mode 100644
index 00000000000..7b6f5ae6d29
--- /dev/null
+++ b/pkg/tsdb/elasticsearch/instrumentation/instrumentation.go
@@ -0,0 +1,33 @@
+package instrumentation
+
+import (
+	"context"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	pluginParsingResponseDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "grafana",
+		Name:      "elasticsearch_plugin_parse_response_duration_seconds",
+		Help:      "Duration of Elasticsearch parsing the response in seconds",
+		Buckets:   []float64{.001, 0.0025, .005, .0075, .01, .02, .03, .04, .05, .075, .1, .25, .5, 1, 5, 10, 25},
+	}, []string{"status", "endpoint"})
+)
+
+const (
+	EndpointQueryData = "queryData"
+)
+
+func UpdatePluginParsingResponseDurationSeconds(ctx context.Context, duration time.Duration, status string) {
+	histogram := pluginParsingResponseDurationSeconds.WithLabelValues(status, EndpointQueryData)
+
+	if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" {
+		histogram.(prometheus.ExemplarObserver).ObserveWithExemplar(duration.Seconds(), prometheus.Labels{"traceID": traceID})
+	} else {
+		histogram.Observe(duration.Seconds())
+	}
+}
diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go
index 3063fb2e0fe..696f1a7f11e 100644
--- a/pkg/tsdb/elasticsearch/response_parser.go
+++ b/pkg/tsdb/elasticsearch/response_parser.go
@@ -20,6 +20,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
+	"github.com/grafana/grafana/pkg/tsdb/elasticsearch/instrumentation"
 )
 
 const (
@@ -109,8 +110,9 @@ func parseResponse(ctx context.Context, responses []*es.SearchResponse, targets
 				span.SetStatus(codes.Error, err.Error())
 				resSpan.RecordError(err)
 				resSpan.SetStatus(codes.Error, err.Error())
+				logger.Error("Error processing buckets", "error", err, "query", string(mt), "aggregationsLength", len(res.Aggregations), "stage", es.StageParseResponse)
+				instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "error")
 				resSpan.End()
-				logger.Error("Error processing buckets", "error", err, "query", string(mt), "aggregationsLength", len(res.Aggregations))
 				return &backend.QueryDataResponse{}, err
 			}
 			nameFields(queryRes, target)
@@ -118,8 +120,9 @@ func parseResponse(ctx context.Context, responses []*es.SearchResponse, targets
 
 			result.Responses[target.RefID] = queryRes
 		}
-		resSpan.End()
+		instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "ok")
 		logger.Info("Finished processing of response", "duration", time.Since(start), "stage", es.StageParseResponse)
+		resSpan.End()
 	}
 	return &result, nil
 }