diff --git a/conf/defaults.ini b/conf/defaults.ini index a56004a53a6..ea10ff1ce9f 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1240,6 +1240,9 @@ state_periodic_save_interval = 5m # Rules will evaluate in sync. disable_jitter = false +# Retention period for Alertmanager notification log entries. +notification_log_retention = 5d + [unified_alerting.screenshots] # Enable screenshots in notifications. You must have either installed the Grafana image rendering # plugin, or set up Grafana to use a remote rendering service. diff --git a/conf/sample.ini b/conf/sample.ini index f315784fed2..1f40a3eab59 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -1156,6 +1156,9 @@ # Rules will evaluate in sync. ;disable_jitter = false +# Retention period for Alertmanager notification log entries. +;notification_log_retention = 5d + [unified_alerting.reserved_labels] # Comma-separated list of reserved labels added by the Grafana Alerting engine that should be disabled. # For example: `disabled_labels=grafana_folder` diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 85eb84fbc20..8755eb2baa9 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -26,13 +26,12 @@ import ( ) const ( - // maintenanceNotificationAndSilences how often should we flush and garbage collect notifications - notificationLogMaintenanceInterval = 15 * time.Minute -) + // How often we flush and garbage collect notifications and silences. + maintenanceInterval = 15 * time.Minute -// How long should we keep silences and notification entries on-disk after they've served their purpose. -var retentionNotificationsAndSilences = 5 * 24 * time.Hour -var silenceMaintenanceInterval = 15 * time.Minute + // How long we keep silences in the kvstore after they've expired. + silenceRetention = 5 * 24 * time.Hour +) type AlertingStore interface { store.AlertingStore @@ -104,8 +103,8 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A silencesOptions := maintenanceOptions{ initialState: silences, - retention: retentionNotificationsAndSilences, - maintenanceFrequency: silenceMaintenanceInterval, + retention: silenceRetention, + maintenanceFrequency: maintenanceInterval, maintenanceFunc: func(state alertingNotify.State) (int64, error) { // Detached context here is to make sure that when the service is shut down the persist operation is executed. return stateStore.SaveSilences(context.Background(), state) @@ -114,8 +113,8 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A nflogOptions := maintenanceOptions{ initialState: nflog, - retention: retentionNotificationsAndSilences, - maintenanceFrequency: notificationLogMaintenanceInterval, + retention: cfg.UnifiedAlerting.NotificationLogRetention, + maintenanceFrequency: maintenanceInterval, maintenanceFunc: func(state alertingNotify.State) (int64, error) { // Detached context here is to make sure that when the service is shut down the persist operation is executed. return stateStore.SaveNotificationLog(context.Background(), state) diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index eaca5d11afc..3a6f53c28a3 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -101,6 +101,9 @@ type UnifiedAlertingSettings struct { MaxStateSaveConcurrency int StatePeriodicSaveInterval time.Duration RulesPerRuleGroupLimit int64 + + // Retention period for Alertmanager notification log entries. + NotificationLogRetention time.Duration } // RemoteAlertmanagerSettings contains the configuration needed @@ -173,6 +176,8 @@ func (cfg *Cfg) readUnifiedAlertingEnabledSetting(section *ini.Section) (*bool, // ReadUnifiedAlertingSettings reads both the `unified_alerting` and `alerting` sections of the configuration while preferring configuration the `alerting` section. // It first reads the `unified_alerting` section, then looks for non-defaults on the `alerting` section and prefers those. +// +// nolint: gocyclo func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error { var err error uaCfg := UnifiedAlertingSettings{} @@ -378,6 +383,11 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error { return err } + uaCfg.NotificationLogRetention, err = gtime.ParseDuration(valueAsString(ua, "notification_log_retention", (5 * 24 * time.Hour).String())) + if err != nil { + return err + } + cfg.UnifiedAlerting = uaCfg return nil }