Alerting: Add backend support for keep_firing_for (#100750)

What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert

pkg/services/ngalert/api/validation/api_ruler_validation.go

@@ -157,6 +157,11 @@ func validateAlertingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
 		return ngmodels.AlertRule{}, err
 	}
 
+	newRule.KeepFiringFor, err = validateKeepFiringForInterval(in)
+	if err != nil {
+		return ngmodels.AlertRule{}, err
+	}
+
 	return newRule, nil
 }
 
@@ -185,6 +190,7 @@ func validateRecordingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
 	newRule.ExecErrState = ""
 	newRule.Condition = ""
 	newRule.For = 0
+	newRule.KeepFiringFor = 0
 	newRule.NotificationSettings = nil
 
 	return newRule, nil
@@ -272,6 +278,21 @@ func validateForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Dur
 	return duration, nil
 }
 
+// validateKeepFiringForInterval validates ApiRuleNode.KeepFiringFor and converts it to time.Duration. If the field is not specified returns 0 if GrafanaManagedAlert.UID is empty and -1 if it is not.
+func validateKeepFiringForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Duration, error) {
+	if ruleNode.ApiRuleNode == nil || ruleNode.ApiRuleNode.KeepFiringFor == nil {
+		if ruleNode.GrafanaManagedAlert.UID != "" {
+			return -1, nil // will be patched later with the real value of the current version of the rule
+		}
+		return 0, nil // if it's a new rule, use the 0 as the default
+	}
+	duration := time.Duration(*ruleNode.ApiRuleNode.KeepFiringFor)
+	if duration < 0 {
+		return 0, fmt.Errorf("field `keep_firing_for` cannot be negative [%v]. only 0 or any positive value is allowed", *ruleNode.ApiRuleNode.KeepFiringFor)
+	}
+	return duration, nil
+}
+
 // ValidateRuleGroup validates API model (definitions.PostableRuleGroupConfig) and converts it to a collection of models.AlertRule.
 // Returns a slice that contains all rules described by API model or error if either group specification or an alert definition is not valid.
 // It also returns a map containing current existing alerts that don't contain the is_paused field in the body of the call.