From ed42119907d9d172a0f9758d7ac8f9b976546ddb Mon Sep 17 00:00:00 2001 From: Steve Simpson Date: Fri, 24 May 2024 23:03:34 +0200 Subject: [PATCH] Alerting: Pass metrics Registerer into NewExternalAlertmanagerSender. (#88313) * Alerting: Pass metrics Registerer into NewExternalAlertmanagerSender. I will work on a separate change to export the metrics from Grafana, this is a little more complicated. * Typo --- pkg/services/ngalert/remote/alertmanager.go | 3 ++- pkg/services/ngalert/sender/router.go | 3 ++- pkg/services/ngalert/sender/sender.go | 4 ++-- pkg/services/ngalert/sender/sender_test.go | 5 +++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 4c69003113e..cbdf5fb883a 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -17,6 +17,7 @@ import ( amgeneral "github.com/prometheus/alertmanager/api/v2/client/general" amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver" amsilence "github.com/prometheus/alertmanager/api/v2/client/silence" + "github.com/prometheus/client_golang/prometheus" alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb" alertingNotify "github.com/grafana/alerting/notify" @@ -123,7 +124,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt return c.Do(req.WithContext(ctx)) } senderLogger := log.New("ngalert.sender.external-alertmanager") - s := sender.NewExternalAlertmanagerSender(senderLogger, sender.WithDoFunc(doFunc)) + s := sender.NewExternalAlertmanagerSender(senderLogger, prometheus.NewRegistry(), sender.WithDoFunc(doFunc)) s.Run() err = s.ApplyConfig(cfg.OrgID, 0, []sender.ExternalAMcfg{{URL: cfg.URL + "/alertmanager"}}) if err != nil { diff --git a/pkg/services/ngalert/sender/router.go b/pkg/services/ngalert/sender/router.go index 3da5ae30846..1d0c5aeafa7 100644 --- a/pkg/services/ngalert/sender/router.go +++ b/pkg/services/ngalert/sender/router.go @@ -12,6 +12,7 @@ import ( "time" "github.com/benbjohnson/clock" + "github.com/prometheus/client_golang/prometheus" "github.com/grafana/grafana/pkg/api/datasource" "github.com/grafana/grafana/pkg/infra/log" @@ -164,7 +165,7 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase(ctx context.Context) error // No sender and have Alertmanager(s) to send to - start a new one. d.logger.Info("Creating new sender for the external alertmanagers", "org", cfg.OrgID, "alertmanagers", redactedAMs) senderLogger := log.New("ngalert.sender.external-alertmanager") - s := NewExternalAlertmanagerSender(senderLogger) + s := NewExternalAlertmanagerSender(senderLogger, prometheus.NewRegistry()) d.externalAlertmanagers[cfg.OrgID] = s s.Run() diff --git a/pkg/services/ngalert/sender/sender.go b/pkg/services/ngalert/sender/sender.go index 62d08281a6d..23ad4e7df0f 100644 --- a/pkg/services/ngalert/sender/sender.go +++ b/pkg/services/ngalert/sender/sender.go @@ -80,7 +80,7 @@ func (cfg *ExternalAMcfg) headerString() string { return result.String() } -func NewExternalAlertmanagerSender(l log.Logger, opts ...Option) *ExternalAlertmanager { +func NewExternalAlertmanagerSender(l log.Logger, reg prometheus.Registerer, opts ...Option) *ExternalAlertmanager { sdCtx, sdCancel := context.WithCancel(context.Background()) s := &ExternalAlertmanager{ logger: l, @@ -90,7 +90,7 @@ func NewExternalAlertmanagerSender(l log.Logger, opts ...Option) *ExternalAlertm s.manager = NewManager( // Injecting a new registry here means these metrics are not exported. // Once we fix the individual Alertmanager metrics we should fix this scenario too. - &Options{QueueCapacity: defaultMaxQueueCapacity, Registerer: prometheus.NewRegistry()}, + &Options{QueueCapacity: defaultMaxQueueCapacity, Registerer: reg}, s.logger, ) diff --git a/pkg/services/ngalert/sender/sender_test.go b/pkg/services/ngalert/sender/sender_test.go index 177e2139350..add647dbc42 100644 --- a/pkg/services/ngalert/sender/sender_test.go +++ b/pkg/services/ngalert/sender/sender_test.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/prometheus/alertmanager/api/v2/models" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" ) @@ -50,7 +51,7 @@ func TestSanitizeLabelName(t *testing.T) { for _, tc := range cases { logger := log.New("ngalert.sender.external-alertmanager") - am := NewExternalAlertmanagerSender(logger) + am := NewExternalAlertmanagerSender(logger, prometheus.NewRegistry()) t.Run(tc.desc, func(t *testing.T) { res, err := am.sanitizeLabelName(tc.labelName) @@ -98,7 +99,7 @@ func TestSanitizeLabelSet(t *testing.T) { for _, tc := range cases { logger := log.New("ngalert.sender.external-alertmanager") - am := NewExternalAlertmanagerSender(logger) + am := NewExternalAlertmanagerSender(logger, prometheus.NewRegistry()) t.Run(tc.desc, func(t *testing.T) { require.Equal(t, tc.expectedResult, am.sanitizeLabelSet(tc.labelset)) })