diff --git a/pkg/services/annotations/annotationsimpl/loki/historian_store.go b/pkg/services/annotations/annotationsimpl/loki/historian_store.go index 8784e6c4141..b77e96f8320 100644 --- a/pkg/services/annotations/annotationsimpl/loki/historian_store.go +++ b/pkg/services/annotations/annotationsimpl/loki/historian_store.go @@ -67,8 +67,9 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, db d return nil } + metrics := ngmetrics.NewHistorianMetrics(reg, subsystem) return &LokiHistorianStore{ - client: lokiclient.NewLokiClient(lokiCfg, lokiclient.NewRequester(), ngmetrics.NewHistorianMetrics(reg, subsystem), log, tracer), + client: lokiclient.NewLokiClient(lokiCfg, lokiclient.NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log, tracer, historian.LokiClientSpanName), db: db, log: log, ruleStore: ruleStore, diff --git a/pkg/services/ngalert/lokiclient/client.go b/pkg/services/ngalert/lokiclient/client.go index fd67eb655da..08d23b73797 100644 --- a/pkg/services/ngalert/lokiclient/client.go +++ b/pkg/services/ngalert/lokiclient/client.go @@ -11,10 +11,11 @@ import ( "strconv" "time" + "github.com/grafana/dskit/instrument" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/ngalert/client" - "github.com/grafana/grafana/pkg/services/ngalert/metrics" + "github.com/prometheus/client_golang/prometheus" ) const defaultPageSize = 1000 @@ -45,11 +46,12 @@ type LokiConfig struct { } type HttpLokiClient struct { - client client.Requester - encoder encoder - cfg LokiConfig - metrics *metrics.Historian - log log.Logger + client client.Requester + encoder encoder + cfg LokiConfig + bytesWritten prometheus.Counter + writeDuration *instrument.HistogramCollector + log log.Logger } // Kind of Operation (=, !=, =~, !~) @@ -66,15 +68,15 @@ const ( NeqRegEx Operator = "!~" ) -func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger, tracer tracing.Tracer) *HttpLokiClient { - tc := client.NewTimedClient(req, metrics.WriteDuration) - trc := client.NewTracedClient(tc, tracer, "ngalert.historian.client") +func NewLokiClient(cfg LokiConfig, req client.Requester, bytesWritten prometheus.Counter, writeDuration *instrument.HistogramCollector, logger log.Logger, tracer tracing.Tracer, spanName string) *HttpLokiClient { + tc := client.NewTimedClient(req, writeDuration) + trc := client.NewTracedClient(tc, tracer, spanName) return &HttpLokiClient{ - client: trc, - encoder: cfg.Encoder, - cfg: cfg, - metrics: metrics, - log: logger.New("protocol", "http"), + client: trc, + encoder: cfg.Encoder, + cfg: cfg, + bytesWritten: bytesWritten, + log: logger.New("protocol", "http"), } } @@ -158,7 +160,7 @@ func (c *HttpLokiClient) Push(ctx context.Context, s []Stream) error { req.Header.Add(k, v) } - c.metrics.BytesWritten.Add(float64(len(enc))) + c.bytesWritten.Add(float64(len(enc))) req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { diff --git a/pkg/services/ngalert/lokiclient/client_test.go b/pkg/services/ngalert/lokiclient/client_test.go index 821d0f79574..0cb5e46a7ad 100644 --- a/pkg/services/ngalert/lokiclient/client_test.go +++ b/pkg/services/ngalert/lokiclient/client_test.go @@ -20,6 +20,8 @@ import ( "github.com/grafana/grafana/pkg/infra/tracing" ) +const lokiClientSpanName = "testLokiClientSpanName" + func TestLokiHTTPClient(t *testing.T) { t.Run("push formats expected data", func(t *testing.T) { req := NewFakeRequester() @@ -137,11 +139,12 @@ func TestLokiHTTPClient_Manual(t *testing.T) { url, err := url.Parse("https://logs-prod-eu-west-0.grafana.net") require.NoError(t, err) + metrics := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem) client := NewLokiClient(LokiConfig{ ReadPathURL: url, WritePathURL: url, Encoder: JsonEncoder{}, - }, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest()) + }, NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName) // Unauthorized request should fail against Grafana Cloud. err = client.Ping(context.Background()) @@ -163,13 +166,14 @@ func TestLokiHTTPClient_Manual(t *testing.T) { url, err := url.Parse("https://logs-prod-eu-west-0.grafana.net") require.NoError(t, err) + metrics := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem) client := NewLokiClient(LokiConfig{ ReadPathURL: url, WritePathURL: url, BasicAuthUser: "", BasicAuthPassword: "", Encoder: JsonEncoder{}, - }, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest()) + }, NewRequester(), metrics.BytesWritten, metrics.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName) // When running on prem, you might need to set the tenant id, // so the x-scope-orgid header is set. @@ -303,7 +307,7 @@ func createTestLokiClient(req client.Requester) *HttpLokiClient { Encoder: JsonEncoder{}, } met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem) - return NewLokiClient(cfg, req, met, log.NewNopLogger(), tracing.InitializeTracerForTest()) + return NewLokiClient(cfg, req, met.BytesWritten, met.WriteDuration, log.NewNopLogger(), tracing.InitializeTracerForTest(), lokiClientSpanName) } func reqBody(t *testing.T, req *http.Request) string { diff --git a/pkg/services/ngalert/state/historian/loki.go b/pkg/services/ngalert/state/historian/loki.go index 2e7f1dde9d0..22f220ec285 100644 --- a/pkg/services/ngalert/state/historian/loki.go +++ b/pkg/services/ngalert/state/historian/loki.go @@ -45,6 +45,7 @@ const ( const ( StateHistoryLabelKey = "from" StateHistoryLabelValue = "state-history" + LokiClientSpanName = "ngalert.historian.client" ) const defaultQueryRange = 6 * time.Hour @@ -88,7 +89,7 @@ type RemoteLokiBackend struct { func NewRemoteLokiBackend(logger log.Logger, cfg lokiclient.LokiConfig, req client.Requester, metrics *metrics.Historian, tracer tracing.Tracer, ruleStore RuleStore, ac AccessControl) *RemoteLokiBackend { return &RemoteLokiBackend{ - client: lokiclient.NewLokiClient(cfg, req, metrics, logger, tracer), + client: lokiclient.NewLokiClient(cfg, req, metrics.BytesWritten, metrics.WriteDuration, logger, tracer, LokiClientSpanName), externalLabels: cfg.ExternalLabels, clock: clock.New(), metrics: metrics,