Alerting: Notifiication history (#107644)

* Add unified_alerting.notification_history to ini files

* Parse notification history settings

* Move Loki client to a separate package

* Loki client: add params for metrics and traces

* add NotificationHistorian

* rm writeDuration

* remove RangeQuery stuff

* wip

* wip

* wip

* wip

* pass notification historian in tests

* unify loki settings

* unify loki settings

* add test

* update grafana/alerting

* make update-workspace

* add feature toggle

* fix configureNotificationHistorian

* Revert "add feature toggle"

This reverts commit de7af8f7

* add feature toggle

* more tests

* RuleUID

* fix metrics test

* met.Info.Set(0)

pkg/setting/setting_unified_alerting.go

@@ -64,6 +64,7 @@ const (
 	// DefaultRuleEvaluationInterval indicates a default interval of for how long a rule should be evaluated to change state from Pending to Alerting
 	DefaultRuleEvaluationInterval          = SchedulerBaseInterval * 6 // == 60 seconds
 	stateHistoryDefaultEnabled             = true
+	notificationHistoryDefaultEnabled      = false
 	lokiDefaultMaxQueryLength              = 721 * time.Hour // 30d1h, matches the default value in Loki
 	defaultRecordingRequestTimeout         = 10 * time.Second
 	lokiDefaultMaxQuerySize                = 65536 // 64kb
@@ -122,6 +123,7 @@ type UnifiedAlertingSettings struct {
 	ReservedLabels                UnifiedAlertingReservedLabelSettings
 	SkipClustering                bool
 	StateHistory                  UnifiedAlertingStateHistorySettings
+	NotificationHistory           UnifiedAlertingNotificationHistorySettings
 	RemoteAlertmanager            RemoteAlertmanagerSettings
 	RecordingRules                RecordingRuleSettings
 	PrometheusConversion          UnifiedAlertingPrometheusConversionSettings
@@ -181,19 +183,24 @@ type UnifiedAlertingPrometheusConversionSettings struct {
 	RuleQueryOffset time.Duration
 }
 
-type UnifiedAlertingStateHistorySettings struct {
-	Enabled       bool
-	Backend       string
+type UnifiedAlertingLokiSettings struct {
 	LokiRemoteURL string
 	LokiReadURL   string
 	LokiWriteURL  string
 	LokiTenantID  string
 	// LokiBasicAuthUsername and LokiBasicAuthPassword are used for basic auth
 	// if one of them is set.
-	LokiBasicAuthPassword         string
-	LokiBasicAuthUsername         string
-	LokiMaxQueryLength            time.Duration
-	LokiMaxQuerySize              int
+	LokiBasicAuthPassword string
+	LokiBasicAuthUsername string
+	LokiMaxQueryLength    time.Duration
+	LokiMaxQuerySize      int
+	ExternalLabels        map[string]string
+}
+
+type UnifiedAlertingStateHistorySettings struct {
+	Enabled                       bool
+	Backend                       string
+	LokiSettings                  UnifiedAlertingLokiSettings
 	PrometheusMetricName          string
 	PrometheusTargetDatasourceUID string
 	PrometheusWriteTimeout        time.Duration
@@ -202,6 +209,11 @@ type UnifiedAlertingStateHistorySettings struct {
 	ExternalLabels                map[string]string
 }
 
+type UnifiedAlertingNotificationHistorySettings struct {
+	Enabled      bool
+	LokiSettings UnifiedAlertingLokiSettings
+}
+
 // IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
 // It hides the implementation details of the Enabled and simplifies its usage.
 func (u *UnifiedAlertingSettings) IsEnabled() bool {
@@ -453,16 +465,18 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	stateHistory := iniFile.Section("unified_alerting.state_history")
 	stateHistoryLabels := iniFile.Section("unified_alerting.state_history.external_labels")
 	uaCfgStateHistory := UnifiedAlertingStateHistorySettings{
-		Enabled:                       stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
-		Backend:                       stateHistory.Key("backend").MustString("annotations"),
-		LokiRemoteURL:                 stateHistory.Key("loki_remote_url").MustString(""),
-		LokiReadURL:                   stateHistory.Key("loki_remote_read_url").MustString(""),
-		LokiWriteURL:                  stateHistory.Key("loki_remote_write_url").MustString(""),
-		LokiTenantID:                  stateHistory.Key("loki_tenant_id").MustString(""),
-		LokiBasicAuthUsername:         stateHistory.Key("loki_basic_auth_username").MustString(""),
-		LokiBasicAuthPassword:         stateHistory.Key("loki_basic_auth_password").MustString(""),
-		LokiMaxQueryLength:            stateHistory.Key("loki_max_query_length").MustDuration(lokiDefaultMaxQueryLength),
-		LokiMaxQuerySize:              stateHistory.Key("loki_max_query_size").MustInt(lokiDefaultMaxQuerySize),
+		Enabled: stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
+		Backend: stateHistory.Key("backend").MustString("annotations"),
+		LokiSettings: UnifiedAlertingLokiSettings{
+			LokiRemoteURL:         stateHistory.Key("loki_remote_url").MustString(""),
+			LokiReadURL:           stateHistory.Key("loki_remote_read_url").MustString(""),
+			LokiWriteURL:          stateHistory.Key("loki_remote_write_url").MustString(""),
+			LokiTenantID:          stateHistory.Key("loki_tenant_id").MustString(""),
+			LokiBasicAuthUsername: stateHistory.Key("loki_basic_auth_username").MustString(""),
+			LokiBasicAuthPassword: stateHistory.Key("loki_basic_auth_password").MustString(""),
+			LokiMaxQueryLength:    stateHistory.Key("loki_max_query_length").MustDuration(lokiDefaultMaxQueryLength),
+			LokiMaxQuerySize:      stateHistory.Key("loki_max_query_size").MustInt(lokiDefaultMaxQuerySize),
+		},
 		MultiPrimary:                  stateHistory.Key("primary").MustString(""),
 		MultiSecondaries:              splitTrim(stateHistory.Key("secondaries").MustString(""), ","),
 		PrometheusMetricName:          stateHistory.Key("prometheus_metric_name").MustString(defaultHistorianPrometheusMetricName),
@@ -472,6 +486,24 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	}
 	uaCfg.StateHistory = uaCfgStateHistory
 
+	notificationHistory := iniFile.Section("unified_alerting.notification_history")
+	notificationHistoryLabels := iniFile.Section("unified_alerting.notification_history.external_labels")
+	uaCfgNotificationHistory := UnifiedAlertingNotificationHistorySettings{
+		Enabled: notificationHistory.Key("enabled").MustBool(notificationHistoryDefaultEnabled),
+		LokiSettings: UnifiedAlertingLokiSettings{
+			LokiRemoteURL:         notificationHistory.Key("loki_remote_url").MustString(""),
+			LokiReadURL:           notificationHistory.Key("loki_remote_read_url").MustString(""),
+			LokiWriteURL:          notificationHistory.Key("loki_remote_write_url").MustString(""),
+			LokiTenantID:          notificationHistory.Key("loki_tenant_id").MustString(""),
+			LokiBasicAuthUsername: notificationHistory.Key("loki_basic_auth_username").MustString(""),
+			LokiBasicAuthPassword: notificationHistory.Key("loki_basic_auth_password").MustString(""),
+			LokiMaxQueryLength:    notificationHistory.Key("loki_max_query_length").MustDuration(lokiDefaultMaxQueryLength),
+			LokiMaxQuerySize:      notificationHistory.Key("loki_max_query_size").MustInt(lokiDefaultMaxQuerySize),
+			ExternalLabels:        notificationHistoryLabels.KeysHash(),
+		},
+	}
+	uaCfg.NotificationHistory = uaCfgNotificationHistory
+
 	prometheusConversion := iniFile.Section("unified_alerting.prometheus_conversion")
 	uaCfg.PrometheusConversion = UnifiedAlertingPrometheusConversionSettings{
 		RuleQueryOffset: prometheusConversion.Key("rule_query_offset").MustDuration(time.Minute),