From 51d7aa2bef7549b3d3aa9fe08f37a6f03aa1966e Mon Sep 17 00:00:00 2001 From: Santiago Date: Mon, 12 May 2025 10:37:27 +0200 Subject: [PATCH] Remote Alertmanager: Configure SMTP From address (#104925) * Remote Alertmanager: Configure SMTP From address * include smtp from address in config comparison * updte tests * trigger build * make linter happy * trigger build * fix test --- pkg/services/ngalert/ngalert.go | 3 + pkg/services/ngalert/remote/alertmanager.go | 11 +++- .../ngalert/remote/alertmanager_test.go | 57 +++++++++++++------ .../client/alertmanager_configuration.go | 2 + pkg/services/ngalert/remote/client/mimir.go | 3 + 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index cb1898bd2c4..a3ab48b821f 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -213,6 +213,7 @@ func (ng *AlertNG) init() error { PromoteConfig: true, SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval, ExternalURL: ng.Cfg.AppURL, + SmtpFrom: ng.Cfg.Smtp.FromAddress, StaticHeaders: ng.Cfg.Smtp.StaticHeaders, } remoteAM, err := createRemoteAlertmanager(ctx, cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer) @@ -249,6 +250,7 @@ func (ng *AlertNG) init() error { TenantID: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.TenantID, URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL, ExternalURL: ng.Cfg.AppURL, + SmtpFrom: ng.Cfg.Smtp.FromAddress, StaticHeaders: ng.Cfg.Smtp.StaticHeaders, } remoteAM, err := createRemoteAlertmanager(ctx, cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer) @@ -287,6 +289,7 @@ func (ng *AlertNG) init() error { URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL, SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval, ExternalURL: ng.Cfg.AppURL, + SmtpFrom: ng.Cfg.Smtp.FromAddress, StaticHeaders: ng.Cfg.Smtp.StaticHeaders, } remoteAM, err := createRemoteAlertmanager(ctx, cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer) diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index a6767ecafd1..60f63cf8a32 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -60,6 +60,7 @@ type Alertmanager struct { orgID int64 ready bool sender *sender.ExternalAlertmanager + smtpFrom string state stateStore tenantID string url string @@ -86,7 +87,8 @@ type AlertmanagerConfig struct { // The same flag is used for promoting state. PromoteConfig bool - // StaticHeaders are used in email notifications sent by the remote Alertmanager. + // SmtpFrom and StaticHeaders are used in email notifications sent by the remote Alertmanager. + SmtpFrom string StaticHeaders map[string]string // SyncInterval determines how often we should attempt to synchronize configuration. @@ -126,6 +128,7 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto URL: u, PromoteConfig: cfg.PromoteConfig, ExternalURL: cfg.ExternalURL, + SmtpFrom: cfg.SmtpFrom, StaticHeaders: cfg.StaticHeaders, } mc, err := remoteClient.New(mcCfg, metrics, tracer) @@ -193,6 +196,7 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto metrics: metrics, mimirClient: mc, orgID: cfg.OrgID, + smtpFrom: cfg.SmtpFrom, state: store, sender: s, syncInterval: cfg.SyncInterval, @@ -670,6 +674,11 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, hash [16]byte) boo return true } + if rc.SmtpFrom != am.smtpFrom { + am.log.Debug("SMTP 'from' address is different, sending the configuration to the remote Alertmanager", "remote", rc.SmtpFrom, "local", am.smtpFrom) + return true + } + rawRemote, err := json.Marshal(rc.GrafanaAlertmanagerConfig) if err != nil { am.log.Error("Unable to marshal the remote Alertmanager configuration for comparison", "err", err) diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index 7ce1d41e40d..29e29314770 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -137,20 +137,24 @@ func TestApplyConfig(t *testing.T) { }) var configSent client.UserGrafanaConfig - var lastConfigSync, lastStateSync time.Time + var configSyncs, stateSyncs int okHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader)) require.Equal(t, "true", r.Header.Get(client.RemoteAlertmanagerHeader)) + var res = map[string]any{"status": "success"} + if r.Method == http.MethodPost { if strings.Contains(r.URL.Path, "/config") { require.NoError(t, json.NewDecoder(r.Body).Decode(&configSent)) - lastConfigSync = time.Now() + configSyncs++ } else { - lastStateSync = time.Now() + stateSyncs++ } + } else { + res["data"] = &configSent } w.Header().Add("content-type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"})) + require.NoError(t, json.NewEncoder(w).Encode(res)) }) // Encrypt receivers to save secrets in the database. @@ -178,6 +182,7 @@ func TestApplyConfig(t *testing.T) { PromoteConfig: true, SyncInterval: 1 * time.Hour, ExternalURL: "https://test.grafana.com", + SmtpFrom: "test-instance@grafana.net", StaticHeaders: map[string]string{"Header-1": "Value-1", "Header-2": "Value-2"}, } @@ -197,11 +202,15 @@ func TestApplyConfig(t *testing.T) { } require.Error(t, am.ApplyConfig(ctx, config)) require.False(t, am.Ready()) + require.Equal(t, 0, stateSyncs) + require.Equal(t, 0, configSyncs) // A 200 status code response should make the check succeed. server.Config.Handler = okHandler require.NoError(t, am.ApplyConfig(ctx, config)) require.True(t, am.Ready()) + require.Equal(t, 1, stateSyncs) + require.Equal(t, 1, configSyncs) // The sent configuration should be unencrypted and promoted. amCfg, err := json.Marshal(configSent.GrafanaAlertmanagerConfig) @@ -209,28 +218,46 @@ func TestApplyConfig(t *testing.T) { require.JSONEq(t, testGrafanaConfigWithSecret, string(amCfg)) require.True(t, configSent.Promoted) - // Grafana's URL and static headers should be sent alongside the configuration. + // Grafana's URL, email "from" address, and static headers should be sent alongside the configuration. require.Equal(t, cfg.ExternalURL, configSent.ExternalURL) + require.Equal(t, cfg.SmtpFrom, configSent.SmtpFrom) require.Equal(t, cfg.StaticHeaders, configSent.StaticHeaders) // If we already got a 200 status code response and the sync interval hasn't elapsed, // we shouldn't send the state/configuration again. - expStateSync := lastStateSync - expConfigSync := lastConfigSync require.NoError(t, am.ApplyConfig(ctx, config)) - require.Equal(t, expStateSync, lastStateSync) - require.Equal(t, expConfigSync, lastConfigSync) + require.Equal(t, 1, stateSyncs) + require.Equal(t, 1, configSyncs) - // Changing the sync interval and calling ApplyConfig again + // Changing the sync interval and calling ApplyConfig again with a new config // should result in us sending the configuration but not the state. am.syncInterval = 0 + config = &ngmodels.AlertConfiguration{ + AlertmanagerConfiguration: string(testGrafanaConfig), + } require.NoError(t, am.ApplyConfig(ctx, config)) - require.Equal(t, lastStateSync, expStateSync) - require.Greater(t, lastConfigSync, expConfigSync) + require.Equal(t, 2, configSyncs) + require.Equal(t, 1, stateSyncs) + + // After a restart, the Alertmanager shouldn't send the configuration if it has not changed. + am, err = NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest()) + require.NoError(t, err) + require.NoError(t, am.ApplyConfig(ctx, config)) + require.Equal(t, 2, configSyncs) + + // Changing the "from" address should result in the configuration being updated. + cfg.SmtpFrom = "new-address@test.com" + am, err = NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest()) + require.NoError(t, err) + require.NoError(t, am.ApplyConfig(ctx, config)) + require.Equal(t, 3, configSyncs) + require.Equal(t, am.smtpFrom, configSent.SmtpFrom) // Failing to add the auto-generated routes should result in an error. - am.autogenFn = errAutogenFn + am, err = NewAlertmanager(cfg, fstore, secretsService.Decrypt, errAutogenFn, m, tracing.InitializeTracerForTest()) + require.NoError(t, err) require.ErrorIs(t, am.ApplyConfig(ctx, config), errTest) + require.Equal(t, 3, configSyncs) } func TestCompareAndSendConfiguration(t *testing.T) { @@ -315,9 +342,7 @@ func TestCompareAndSendConfiguration(t *testing.T) { "error from autogen function", strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, fmt.Sprintf("%q:%q", "password", base64.StdEncoding.EncodeToString(testValue)), 1), errAutogenFn, - &client.UserGrafanaConfig{ - GrafanaAlertmanagerConfig: cfgWithSecret, - }, + nil, errTest.Error(), }, { diff --git a/pkg/services/ngalert/remote/client/alertmanager_configuration.go b/pkg/services/ngalert/remote/client/alertmanager_configuration.go index d1808819a84..e2137c1feb0 100644 --- a/pkg/services/ngalert/remote/client/alertmanager_configuration.go +++ b/pkg/services/ngalert/remote/client/alertmanager_configuration.go @@ -22,6 +22,7 @@ type UserGrafanaConfig struct { Default bool `json:"default"` Promoted bool `json:"promoted"` ExternalURL string `json:"external_url"` + SmtpFrom string `json:"smtp_from"` StaticHeaders map[string]string `json:"static_headers"` } @@ -56,6 +57,7 @@ func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *apimo Default: isDefault, Promoted: mc.promoteConfig, ExternalURL: mc.externalURL, + SmtpFrom: mc.smtpFrom, StaticHeaders: mc.staticHeaders, }) if err != nil { diff --git a/pkg/services/ngalert/remote/client/mimir.go b/pkg/services/ngalert/remote/client/mimir.go index a75d11beaf8..e2bd3988ead 100644 --- a/pkg/services/ngalert/remote/client/mimir.go +++ b/pkg/services/ngalert/remote/client/mimir.go @@ -48,6 +48,7 @@ type Mimir struct { metrics *metrics.RemoteAlertmanager promoteConfig bool externalURL string + smtpFrom string staticHeaders map[string]string } @@ -59,6 +60,7 @@ type Config struct { Logger log.Logger PromoteConfig bool ExternalURL string + SmtpFrom string StaticHeaders map[string]string } @@ -103,6 +105,7 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer metrics: metrics, promoteConfig: cfg.PromoteConfig, externalURL: cfg.ExternalURL, + smtpFrom: cfg.SmtpFrom, staticHeaders: cfg.StaticHeaders, }, nil }