From 7dc13d63b99e7156f7870c8378726e1f524d8ec5 Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Mon, 19 May 2025 21:40:51 -0400 Subject: [PATCH] Alerting: Refactor Grafana Alertmanager (#105568) * update alerting module * use NotificationsConfiguration * update to use opts and configure new fields * use TenantID instead of orgId --- go.mod | 2 +- go.sum | 4 +- pkg/services/ngalert/notifier/alertmanager.go | 111 +++++++----------- pkg/services/ngalert/notifier/config.go | 60 ---------- .../notifier/multiorg_alertmanager_test.go | 2 +- 5 files changed, 44 insertions(+), 135 deletions(-) diff --git a/go.mod b/go.mod index f847f4fc32e..e798a426820 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/googleapis/go-sql-spanner v1.11.1 // @grafana/grafana-search-and-storage github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // @grafana/grafana-app-platform-squad - github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf // @grafana/alerting-backend + github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00 // @grafana/alerting-backend github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb // @grafana/identity-access-team github.com/grafana/authlib/types v0.0.0-20250325095148-d6da9c164a7d // @grafana/identity-access-team github.com/grafana/dataplane/examples v0.0.1 // @grafana/observability-metrics diff --git a/go.sum b/go.sum index d0d08245d99..cc8e24a93ac 100644 --- a/go.sum +++ b/go.sum @@ -1569,8 +1569,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf h1:cbwO+P4La7EnDY1gdzFxsAf9u8ygM/7Laimuh7lYrNo= -github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf/go.mod h1:pMfhRxL2LZ3Pm8iy7VcVsb9CLYuBtjFYbf1oxgx7yFA= +github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00 h1:i0IpxkT1bX96F5D/oIxDdXLu+Nz/pU4rHlmnswpVLF0= +github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00/go.mod h1:pMfhRxL2LZ3Pm8iy7VcVsb9CLYuBtjFYbf1oxgx7yFA= github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb h1:oTl2j6/4miQUYmXANp2pBuYCWA5f8NVYFfCWpczpFso= github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb/go.mod h1:PBtQaXwkFu4BAt2aXsR7w8p8NVpdjV5aJYhqRDei9Us= github.com/grafana/authlib/types v0.0.0-20250325095148-d6da9c164a7d h1:34E6btDAhdDOiSEyrMaYaHwnJpM8w9QKzVQZIBzLNmM= diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 025481e49be..85666eb353f 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -10,8 +10,6 @@ import ( "time" alertingNotify "github.com/grafana/alerting/notify" - "github.com/grafana/alerting/receivers" - alertingTemplates "github.com/grafana/alerting/templates" "github.com/prometheus/alertmanager/config" amv2 "github.com/prometheus/alertmanager/api/v2/models" @@ -51,14 +49,10 @@ type alertmanager struct { Base *alertingNotify.GrafanaAlertmanager logger log.Logger - ConfigMetrics *metrics.AlertmanagerConfigMetrics - Settings *setting.Cfg - Store AlertingStore - stateStore stateStore - NotificationService notifications.Service - - decryptFn alertingNotify.GetDecryptedValueFn - orgID int64 + ConfigMetrics *metrics.AlertmanagerConfigMetrics + Store AlertingStore + stateStore stateStore + DefaultConfiguration string } // maintenanceOptions represent the options for components that need maintenance on a frequency within the Alertmanager. @@ -122,7 +116,7 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A } l := log.New("ngalert.notifier.alertmanager", "org", orgID) - amcfg := &alertingNotify.GrafanaAlertmanagerConfig{ + opts := alertingNotify.GrafanaAlertmanagerOpts{ ExternalURL: cfg.AppURL, AlertStoreCallback: nil, PeerTimeout: cfg.UnifiedAlerting.HAPeerTimeout, @@ -132,23 +126,30 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A MaxSilences: cfg.UnifiedAlerting.AlertmanagerMaxSilencesCount, MaxSilenceSizeBytes: cfg.UnifiedAlerting.AlertmanagerMaxSilenceSizeBytes, }, + EmailSender: &emailSender{ns}, + ImageProvider: newImageProvider(store, log.New("ngalert.notifier.image-provider")), + Decrypter: decryptFn, + LoggerFactory: LoggerFactory, + Version: setting.BuildVersion, + TenantKey: "orgID", + TenantID: orgID, + Peer: peer, + Logger: l, + Metrics: alertingNotify.NewGrafanaAlertmanagerMetrics(m.Registerer, l), } - gam, err := alertingNotify.NewGrafanaAlertmanager("orgID", orgID, amcfg, peer, l, alertingNotify.NewGrafanaAlertmanagerMetrics(m.Registerer, l)) + gam, err := alertingNotify.NewGrafanaAlertmanager(opts) if err != nil { return nil, err } am := &alertmanager{ - Base: gam, - ConfigMetrics: m.AlertmanagerConfigMetrics, - Settings: cfg, - Store: store, - NotificationService: ns, - orgID: orgID, - decryptFn: decryptFn, - stateStore: stateStore, - logger: l, + Base: gam, + ConfigMetrics: m.AlertmanagerConfigMetrics, + DefaultConfiguration: cfg.UnifiedAlerting.DefaultConfiguration, + Store: store, + stateStore: stateStore, + logger: l, } return am, nil @@ -171,14 +172,14 @@ func (am *alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error { var outerErr error am.Base.WithLock(func() { cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{ - AlertmanagerConfiguration: am.Settings.UnifiedAlerting.DefaultConfiguration, + AlertmanagerConfiguration: am.DefaultConfiguration, Default: true, ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion), - OrgID: am.orgID, + OrgID: am.Base.TenantID(), LastApplied: time.Now().UTC().Unix(), } - cfg, err := Load([]byte(am.Settings.UnifiedAlerting.DefaultConfiguration)) + cfg, err := Load([]byte(am.DefaultConfiguration)) if err != nil { outerErr = err return @@ -213,7 +214,7 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{ AlertmanagerConfiguration: string(rawConfig), ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion), - OrgID: am.orgID, + OrgID: am.Base.TenantID(), LastApplied: time.Now().UTC().Unix(), } @@ -254,7 +255,7 @@ func (am *alertmanager) ApplyConfig(ctx context.Context, dbCfg *ngmodels.AlertCo return } markConfigCmd := ngmodels.MarkConfigurationAsAppliedCmd{ - OrgID: am.orgID, + OrgID: am.Base.TenantID(), ConfigurationHash: dbCfg.ConfigurationHash, } err = am.Store.MarkConfigurationAsApplied(ctx, &markConfigCmd) @@ -283,11 +284,11 @@ func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig, c am.ConfigMetrics.ObjectMatchers.Set(float64(amu.ObjectMatchers)) am.ConfigMetrics.ConfigHash. - WithLabelValues(strconv.FormatInt(am.orgID, 10)). + WithLabelValues(strconv.FormatInt(am.Base.TenantID(), 10)). Set(hashAsMetricValue(am.Base.ConfigHash())) am.ConfigMetrics.ConfigSizeBytes. - WithLabelValues(strconv.FormatInt(am.orgID, 10)). + WithLabelValues(strconv.FormatInt(am.Base.TenantID(), 10)). Set(float64(cfgSize)) } @@ -316,7 +317,7 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu // It returns a boolean indicating whether the user config was changed and an error. // It is not safe to call concurrently. func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, skipInvalid bool) (bool, error) { - err := AddAutogenConfig(ctx, am.logger, am.Store, am.orgID, &cfg.AlertmanagerConfig, skipInvalid) + err := AddAutogenConfig(ctx, am.logger, am.Store, am.Base.TenantID(), &cfg.AlertmanagerConfig, skipInvalid) if err != nil { return false, err } @@ -336,16 +337,16 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable } am.logger.Info("Applying new configuration to Alertmanager", "configHash", fmt.Sprintf("%x", configHash)) - err = am.Base.ApplyConfig(AlertingConfiguration{ - rawAlertmanagerConfig: rawConfig, - configHash: configHash, - route: cfg.AlertmanagerConfig.Route.AsAMRoute(), - inhibitRules: cfg.AlertmanagerConfig.InhibitRules, - muteTimeIntervals: cfg.AlertmanagerConfig.MuteTimeIntervals, - timeIntervals: cfg.AlertmanagerConfig.TimeIntervals, - templates: ToTemplateDefinitions(cfg), - receivers: PostableApiAlertingConfigToApiReceivers(cfg.AlertmanagerConfig), - receiverIntegrationsFunc: am.buildReceiverIntegrations, + err = am.Base.ApplyConfig(alertingNotify.NotificationsConfiguration{ + RoutingTree: cfg.AlertmanagerConfig.Route.AsAMRoute(), + InhibitRules: cfg.AlertmanagerConfig.InhibitRules, + MuteTimeIntervals: cfg.AlertmanagerConfig.MuteTimeIntervals, + TimeIntervals: cfg.AlertmanagerConfig.TimeIntervals, + Templates: ToTemplateDefinitions(cfg), + Receivers: PostableApiAlertingConfigToApiReceivers(cfg.AlertmanagerConfig), + DispatcherLimits: &nilLimits{}, + Raw: rawConfig, + Hash: configHash, }) if err != nil { return false, err @@ -355,38 +356,6 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable return true, nil } -func (am *alertmanager) AppURL() string { - return am.Settings.AppURL -} - -// buildReceiverIntegrations builds a list of integration notifiers off of a receiver config. -func (am *alertmanager) buildReceiverIntegrations(receiver *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) { - receiverCfg, err := alertingNotify.BuildReceiverConfiguration(context.Background(), receiver, alertingNotify.DecodeSecretsFromBase64, am.decryptFn) - if err != nil { - return nil, err - } - s := &emailSender{am.NotificationService} - img := newImageProvider(am.Store, log.New("ngalert.notifier.image-provider")) - integrations, err := alertingNotify.BuildReceiverIntegrations( - receiverCfg, - tmpl, - img, - LoggerFactory, - func(n receivers.Metadata) (receivers.EmailSender, error) { - return s, nil - }, - func(_ string, n alertingNotify.Notifier) alertingNotify.Notifier { - return n - }, - am.orgID, - setting.BuildVersion, - ) - if err != nil { - return nil, err - } - return integrations, nil -} - // PutAlerts receives the alerts and then sends them through the corresponding route based on whenever the alert has a receiver embedded or not func (am *alertmanager) PutAlerts(_ context.Context, postableAlerts apimodels.PostableAlerts) error { alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts)) diff --git a/pkg/services/ngalert/notifier/config.go b/pkg/services/ngalert/notifier/config.go index bffe1b72ce3..323c40c5263 100644 --- a/pkg/services/ngalert/notifier/config.go +++ b/pkg/services/ngalert/notifier/config.go @@ -4,9 +4,6 @@ import ( "encoding/json" "fmt" - alertingNotify "github.com/grafana/alerting/notify" - alertingTemplates "github.com/grafana/alerting/templates" - api "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ) @@ -19,60 +16,3 @@ func Load(rawConfig []byte) (*api.PostableUserConfig, error) { return cfg, nil } - -// AlertingConfiguration provides configuration for an Alertmanager. -// It implements the notify.Configuration interface. -type AlertingConfiguration struct { - route *alertingNotify.Route - inhibitRules []alertingNotify.InhibitRule - muteTimeIntervals []alertingNotify.MuteTimeInterval - timeIntervals []alertingNotify.TimeInterval - templates []alertingTemplates.TemplateDefinition - rawAlertmanagerConfig []byte - configHash [16]byte - - receivers []*alertingNotify.APIReceiver - receiverIntegrationsFunc func(r *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) -} - -func (a AlertingConfiguration) BuildReceiverIntegrationsFunc() func(next *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) { - return func(next *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) { - return a.receiverIntegrationsFunc(next, tmpl) - } -} - -func (a AlertingConfiguration) DispatcherLimits() alertingNotify.DispatcherLimits { - return &nilLimits{} -} - -func (a AlertingConfiguration) InhibitRules() []alertingNotify.InhibitRule { - return a.inhibitRules -} - -func (a AlertingConfiguration) MuteTimeIntervals() []alertingNotify.MuteTimeInterval { - return a.muteTimeIntervals -} - -func (a AlertingConfiguration) TimeIntervals() []alertingNotify.TimeInterval { - return a.timeIntervals -} - -func (a AlertingConfiguration) Receivers() []*alertingNotify.APIReceiver { - return a.receivers -} - -func (a AlertingConfiguration) RoutingTree() *alertingNotify.Route { - return a.route -} - -func (a AlertingConfiguration) Templates() []alertingTemplates.TemplateDefinition { - return a.templates -} - -func (a AlertingConfiguration) Hash() [16]byte { - return a.configHash -} - -func (a AlertingConfiguration) Raw() []byte { - return a.rawAlertmanagerConfig -} diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index 2fb610d3d5a..92c0209aa37 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -221,7 +221,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { status, err := am.GetStatus(ctx) require.NoError(t, err) require.Equal(t, "N/A", *status.VersionInfo.Version) - require.Equal(t, int64(2), internalAm.orgID) + require.Equal(t, int64(2), internalAm.Base.TenantID()) } // Let's now remove the previous queried organization.