Loki client: add params for metrics and traces

pkg/services/annotations/annotationsimpl/loki/historian_store.go

@@ -67,8 +67,9 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, db d
 		return nil
 	}
 
+	metrics := ngmetrics.NewHistorianMetrics(reg, subsystem)
 	return &LokiHistorianStore{
-		client:    lokiclient.NewLokiClient(lokiCfg, lokiclient.NewRequester(), ngmetrics.NewHistorianMetrics(reg, subsystem), log, tracer),
+		client:    lokiclient.NewLokiClient(lokiCfg, lokiclient.NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log, tracer, historian.LokiClientSpanName),
 		db:        db,
 		log:       log,
 		ruleStore: ruleStore,

pkg/services/ngalert/lokiclient/client.go

@@ -11,10 +11,11 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/grafana/dskit/instrument"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/client"
-	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const defaultPageSize = 1000
@@ -45,11 +46,12 @@ type LokiConfig struct {
 }
 
 type HttpLokiClient struct {
-	client  client.Requester
-	encoder encoder
-	cfg     LokiConfig
-	metrics *metrics.Historian
-	log     log.Logger
+	client        client.Requester
+	encoder       encoder
+	cfg           LokiConfig
+	bytesWritten  prometheus.Counter
+	writeDuration *instrument.HistogramCollector
+	log           log.Logger
 }
 
 // Kind of Operation (=, !=, =~, !~)
@@ -66,15 +68,15 @@ const (
 	NeqRegEx Operator = "!~"
 )
 
-func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger, tracer tracing.Tracer) *HttpLokiClient {
-	tc := client.NewTimedClient(req, metrics.WriteDuration)
-	trc := client.NewTracedClient(tc, tracer, "ngalert.historian.client")
+func NewLokiClient(cfg LokiConfig, req client.Requester, bytesWritten prometheus.Counter, writeDuration *instrument.HistogramCollector, logger log.Logger, tracer tracing.Tracer, spanName string) *HttpLokiClient {
+	tc := client.NewTimedClient(req, writeDuration)
+	trc := client.NewTracedClient(tc, tracer, spanName)
 	return &HttpLokiClient{
-		client:  trc,
-		encoder: cfg.Encoder,
-		cfg:     cfg,
-		metrics: metrics,
-		log:     logger.New("protocol", "http"),
+		client:       trc,
+		encoder:      cfg.Encoder,
+		cfg:          cfg,
+		bytesWritten: bytesWritten,
+		log:          logger.New("protocol", "http"),
 	}
 }
 
@@ -158,7 +160,7 @@ func (c *HttpLokiClient) Push(ctx context.Context, s []Stream) error {
 		req.Header.Add(k, v)
 	}
 
-	c.metrics.BytesWritten.Add(float64(len(enc)))
+	c.bytesWritten.Add(float64(len(enc)))
 	req = req.WithContext(ctx)
 	resp, err := c.client.Do(req)
 	if err != nil {

pkg/services/ngalert/lokiclient/client_test.go

@@ -20,6 +20,8 @@ import (
 	"github.com/grafana/grafana/pkg/infra/tracing"
 )
 
+const lokiClientSpanName = "testLokiClientSpanName"
+
 func TestLokiHTTPClient(t *testing.T) {
 	t.Run("push formats expected data", func(t *testing.T) {
 		req := NewFakeRequester()
@@ -137,11 +139,12 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
 		url, err := url.Parse("https://logs-prod-eu-west-0.grafana.net")
 		require.NoError(t, err)
 
+		metrics := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		client := NewLokiClient(LokiConfig{
 			ReadPathURL:  url,
 			WritePathURL: url,
 			Encoder:      JsonEncoder{},
-		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
+		}, NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName)
 
 		// Unauthorized request should fail against Grafana Cloud.
 		err = client.Ping(context.Background())
@@ -163,13 +166,14 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
 		url, err := url.Parse("https://logs-prod-eu-west-0.grafana.net")
 		require.NoError(t, err)
 
+		metrics := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		client := NewLokiClient(LokiConfig{
 			ReadPathURL:       url,
 			WritePathURL:      url,
 			BasicAuthUser:     "<your_username>",
 			BasicAuthPassword: "<your_password>",
 			Encoder:           JsonEncoder{},
-		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
+		}, NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName)
 
 		// When running on prem, you might need to set the tenant id,
 		// so the x-scope-orgid header is set.
@@ -303,7 +307,7 @@ func createTestLokiClient(req client.Requester) *HttpLokiClient {
 		Encoder:      JsonEncoder{},
 	}
 	met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
-	return NewLokiClient(cfg, req, met, log.NewNopLogger(), tracing.InitializeTracerForTest())
+	return NewLokiClient(cfg, req, met.BytesWritten, met.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName)
 }
 
 func reqBody(t *testing.T, req *http.Request) string {

pkg/services/ngalert/state/historian/loki.go

@@ -45,6 +45,7 @@ const (
 const (
 	StateHistoryLabelKey   = "from"
 	StateHistoryLabelValue = "state-history"
+	LokiClientSpanName     = "ngalert.historian.client"
 )
 
 const defaultQueryRange = 6 * time.Hour
@@ -88,7 +89,7 @@ type RemoteLokiBackend struct {
 
 func NewRemoteLokiBackend(logger log.Logger, cfg lokiclient.LokiConfig, req client.Requester, metrics *metrics.Historian, tracer tracing.Tracer, ruleStore RuleStore, ac AccessControl) *RemoteLokiBackend {
 	return &RemoteLokiBackend{
-		client:         lokiclient.NewLokiClient(cfg, req, metrics, logger, tracer),
+		client:         lokiclient.NewLokiClient(cfg, req, metrics.BytesWritten, metrics.WriteDuration, logger, tracer, LokiClientSpanName),
 		externalLabels: cfg.ExternalLabels,
 		clock:          clock.New(),
 		metrics:        metrics,