From a34e7e176dd2a7a8b86b39e9a1eae5c77c31716d Mon Sep 17 00:00:00 2001
From: owensmallwood <owen.smallwood@grafana.com>
Date: Wed, 12 Feb 2025 15:22:18 -0700
Subject: [PATCH] Unified Storage: Sprinkles latency metric (#100542)

* add sprinkles latency metric

* fixes failing tests - forgot to register metric only once
---
 .../unified/resource/bleve_index_metrics.go   | 35 +++++++++++++++++--
 pkg/storage/unified/sql/server.go             |  5 +++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/pkg/storage/unified/resource/bleve_index_metrics.go b/pkg/storage/unified/resource/bleve_index_metrics.go
index caa0499bcb8..0cb52455cff 100644
--- a/pkg/storage/unified/resource/bleve_index_metrics.go
+++ b/pkg/storage/unified/resource/bleve_index_metrics.go
@@ -11,8 +11,9 @@ import (
 )
 
 var (
-	onceIndex    sync.Once
-	IndexMetrics *BleveIndexMetrics
+	onceIndex             sync.Once
+	IndexMetrics          *BleveIndexMetrics
+	SprinklesIndexMetrics *SprinklesMetrics
 )
 
 type BleveIndexMetrics struct {
@@ -28,8 +29,30 @@ type BleveIndexMetrics struct {
 	IndexTenants      *prometheus.CounterVec
 }
 
+type SprinklesMetrics struct {
+	SprinklesLatency prometheus.Histogram
+}
+
 var IndexCreationBuckets = []float64{1, 5, 10, 25, 50, 75, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}
 
+func NewSprinklesMetrics() *SprinklesMetrics {
+	onceIndex.Do(func() {
+		SprinklesIndexMetrics = &SprinklesMetrics{
+			SprinklesLatency: prometheus.NewHistogram(prometheus.HistogramOpts{
+				Namespace:                       "index_server",
+				Name:                            "sprinkles_latency_seconds",
+				Help:                            "Time (in seconds) it takes until sprinkles are fetched",
+				Buckets:                         instrument.DefBuckets,
+				NativeHistogramBucketFactor:     1.1, // enable native histograms
+				NativeHistogramMaxBucketNumber:  160,
+				NativeHistogramMinResetDuration: time.Hour,
+			}),
+		}
+	})
+
+	return SprinklesIndexMetrics
+}
+
 func NewIndexMetrics(indexDir string, searchBackend SearchBackend) *BleveIndexMetrics {
 	onceIndex.Do(func() {
 		IndexMetrics = &BleveIndexMetrics{
@@ -79,6 +102,14 @@ func NewIndexMetrics(indexDir string, searchBackend SearchBackend) *BleveIndexMe
 	return IndexMetrics
 }
 
+func (s *SprinklesMetrics) Collect(ch chan<- prometheus.Metric) {
+	s.SprinklesLatency.Collect(ch)
+}
+
+func (s *SprinklesMetrics) Describe(ch chan<- *prometheus.Desc) {
+	s.SprinklesLatency.Describe(ch)
+}
+
 func (s *BleveIndexMetrics) Collect(ch chan<- prometheus.Metric) {
 	s.IndexLatency.Collect(ch)
 	s.IndexCreationTime.Collect(ch)
diff --git a/pkg/storage/unified/sql/server.go b/pkg/storage/unified/sql/server.go
index 2c1f1ffd602..b2cade286f5 100644
--- a/pkg/storage/unified/sql/server.go
+++ b/pkg/storage/unified/sql/server.go
@@ -83,10 +83,15 @@ func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg,
 			InitMinCount:  cfg.IndexMinCount,
 		}
 
+		// Register indexer metrics
 		err = reg.Register(resource.NewIndexMetrics(cfg.IndexPath, opts.Search.Backend))
 		if err != nil {
 			slog.Warn("Failed to register indexer metrics", "error", err)
 		}
+		err = reg.Register(resource.NewSprinklesMetrics())
+		if err != nil {
+			slog.Warn("Failed to register sprinkles metrics", "error", err)
+		}
 	}
 
 	rs, err := resource.NewResourceServer(opts)