diff --git a/pkg/services/ngalert/metrics/historian.go b/pkg/services/ngalert/metrics/historian.go new file mode 100644 index 00000000000..32bcf1d180d --- /dev/null +++ b/pkg/services/ngalert/metrics/historian.go @@ -0,0 +1,23 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/instrument" +) + +type Historian struct { + WriteDuration *instrument.HistogramCollector +} + +func NewHistorianMetrics(r prometheus.Registerer) *Historian { + return &Historian{ + WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "state_history_request_duration_seconds", + Help: "Histogram of request durations to the state history store.", + Buckets: instrument.DefBuckets, + }, instrument.HistogramCollectorBuckets)), + } +} diff --git a/pkg/services/ngalert/metrics/ngalert.go b/pkg/services/ngalert/metrics/ngalert.go index ae1a304ed34..732d7432321 100644 --- a/pkg/services/ngalert/metrics/ngalert.go +++ b/pkg/services/ngalert/metrics/ngalert.go @@ -29,6 +29,7 @@ type NGAlert struct { stateMetrics *State multiOrgAlertmanagerMetrics *MultiOrgAlertmanager apiMetrics *API + historianMetrics *Historian } // NewNGAlert manages the metrics of all the alerting components. @@ -39,6 +40,7 @@ func NewNGAlert(r prometheus.Registerer) *NGAlert { stateMetrics: NewStateMetrics(r), multiOrgAlertmanagerMetrics: NewMultiOrgAlertmanagerMetrics(r), apiMetrics: NewAPIMetrics(r), + historianMetrics: NewHistorianMetrics(r), } } @@ -67,3 +69,7 @@ func (moa *MultiOrgAlertmanager) RemoveOrgRegistry(id int64) { func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Registerer { return moa.registries.GetOrCreateOrgRegistry(id) } + +func (ng *NGAlert) GetHistorianMetrics() *Historian { + return ng.historianMetrics +} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index dd7e93ac4f4..b2317989c4d 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -210,7 +210,7 @@ func (ng *AlertNG) init() error { Tracer: ng.tracer, } - history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store) + history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics()) if err != nil { return err } @@ -389,7 +389,7 @@ type Historian interface { state.Historian } -func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore) (Historian, error) { +func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian) (Historian, error) { if !cfg.Enabled { return historian.NewNopHistorian(), nil } @@ -402,7 +402,8 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS if err != nil { return nil, fmt.Errorf("invalid remote loki configuration: %w", err) } - backend := historian.NewRemoteLokiBackend(lcfg) + req := historian.NewRequester() + backend := historian.NewRemoteLokiBackend(lcfg, req, met) testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second) defer cancelFunc() diff --git a/pkg/services/ngalert/state/historian/loki.go b/pkg/services/ngalert/state/historian/loki.go index 4158887aa1d..d536f4c7380 100644 --- a/pkg/services/ngalert/state/historian/loki.go +++ b/pkg/services/ngalert/state/historian/loki.go @@ -9,10 +9,12 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/weaveworks/common/http/client" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/eval" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model" @@ -43,14 +45,16 @@ type remoteLokiClient interface { type RemoteLokiBackend struct { client remoteLokiClient externalLabels map[string]string + metrics *metrics.Historian log log.Logger } -func NewRemoteLokiBackend(cfg LokiConfig) *RemoteLokiBackend { +func NewRemoteLokiBackend(cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend { logger := log.New("ngalert.state.historian", "backend", "loki") return &RemoteLokiBackend{ - client: newLokiClient(cfg, logger), + client: newLokiClient(cfg, req, metrics, logger), externalLabels: cfg.ExternalLabels, + metrics: metrics, log: logger, } } diff --git a/pkg/services/ngalert/state/historian/loki_http.go b/pkg/services/ngalert/state/historian/loki_http.go index 37a87e73add..9437fd66201 100644 --- a/pkg/services/ngalert/state/historian/loki_http.go +++ b/pkg/services/ngalert/state/historian/loki_http.go @@ -11,11 +11,19 @@ import ( "time" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/setting" + "github.com/weaveworks/common/http/client" ) const defaultClientTimeout = 30 * time.Second +func NewRequester() client.Requester { + return &http.Client{ + Timeout: defaultClientTimeout, + } +} + type LokiConfig struct { ReadPathURL *url.URL WritePathURL *url.URL @@ -53,7 +61,7 @@ func NewLokiConfig(cfg setting.UnifiedAlertingStateHistorySettings) (LokiConfig, } type httpLokiClient struct { - client http.Client + client client.Requester cfg LokiConfig log log.Logger } @@ -80,13 +88,12 @@ type Selector struct { Value string } -func newLokiClient(cfg LokiConfig, logger log.Logger) *httpLokiClient { +func newLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *httpLokiClient { + tc := client.NewTimedClient(req, metrics.WriteDuration) return &httpLokiClient{ - client: http.Client{ - Timeout: defaultClientTimeout, - }, - cfg: cfg, - log: logger.New("protocol", "http"), + client: tc, + cfg: cfg, + log: logger.New("protocol", "http"), } } diff --git a/pkg/services/ngalert/state/historian/loki_http_test.go b/pkg/services/ngalert/state/historian/loki_http_test.go index 86b856f3a4e..f202b5339a3 100644 --- a/pkg/services/ngalert/state/historian/loki_http_test.go +++ b/pkg/services/ngalert/state/historian/loki_http_test.go @@ -6,7 +6,9 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/log" @@ -83,7 +85,7 @@ func TestLokiHTTPClient(t *testing.T) { client := newLokiClient(LokiConfig{ ReadPathURL: url, WritePathURL: url, - }, log.NewNopLogger()) + }, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger()) // Unauthorized request should fail against Grafana Cloud. err = client.ping(context.Background()) @@ -110,7 +112,7 @@ func TestLokiHTTPClient(t *testing.T) { WritePathURL: url, BasicAuthUser: "", BasicAuthPassword: "", - }, log.NewNopLogger()) + }, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger()) // When running on prem, you might need to set the tenant id, // so the x-scope-orgid header is set.