Alerting: Add backend support for keep_firing_for (#100750)

What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert

pkg/services/ngalert/api/api_prometheus_test.go

@@ -191,6 +191,38 @@ func TestRouteGetAlertStatuses(t *testing.T) {
 }`, string(r.Body()))
 	})
 
+	t.Run("with a recovering alert", func(t *testing.T) {
+		_, fakeAIM, api := setupAPI(t)
+		fakeAIM.GenerateAlertInstances(1, util.GenerateShortUID(), 1, withRecoveringState())
+		req, err := http.NewRequest("GET", "/api/v1/alerts", nil)
+		require.NoError(t, err)
+		c := &contextmodel.ReqContext{Context: &web.Context{Req: req}, SignedInUser: &user.SignedInUser{OrgID: orgID}}
+
+		r := api.RouteGetAlertStatuses(c)
+		require.Equal(t, http.StatusOK, r.Status())
+		require.JSONEq(t, `
+			{
+				"status": "success",
+				"data": {
+					"alerts": [{
+						"labels": {
+							"alertname": "test_title_0",
+							"instance_label": "test",
+							"label": "test"
+						},
+						"annotations": {
+							"annotation": "test"
+						},
+						"state": "Recovering",
+						"activeAt": "0001-01-01T00:00:00Z",
+						"value": "1.1e+00"
+					}]
+				}
+			}`,
+			string(r.Body()),
+		)
+	})
+
 	t.Run("with the inclusion of internal labels", func(t *testing.T) {
 		_, fakeAIM, api := setupAPI(t)
 		fakeAIM.GenerateAlertInstances(orgID, util.GenerateShortUID(), 2)
@@ -251,6 +283,19 @@ func withAlertingState() forEachState {
 	}
 }
 
+func withRecoveringState() forEachState {
+	return func(s *state.State) *state.State {
+		s.State = eval.Recovering
+		s.LatestResult = &state.Evaluation{
+			EvaluationState: eval.Alerting,
+			EvaluationTime:  timeNow(),
+			Values:          map[string]float64{"B": float64(1.1)},
+			Condition:       "B",
+		}
+		return s
+	}
+}
+
 func withAlertingErrorState() forEachState {
 	return func(s *state.State) *state.State {
 		s.SetAlerting("", timeNow(), timeNow().Add(5*time.Minute))
@@ -347,6 +392,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"type": "alerting",
 				"lastEvaluation": "2022-03-10T14:01:00Z",
 				"duration": 180,
+				"keepFiringFor": 10,
 				"evaluationTime": 60
 			}],
 			"totals": {
@@ -416,6 +462,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"type": "alerting",
 				"lastEvaluation": "2022-03-10T14:01:00Z",
 				"duration": 180,
+				"keepFiringFor": 10,
 				"evaluationTime": 60
 			}],
 			"totals": {
@@ -478,6 +525,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"type": "alerting",
 				"lastEvaluation": "2022-03-10T14:01:00Z",
 				"duration": 180,
+				"keepFiringFor": 10,
 				"evaluationTime": 60
 			}],
 			"totals": {
@@ -495,6 +543,103 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 `, folder.Fullpath), string(r.Body()))
 	})
 
+	t.Run("with a recovering alert", func(t *testing.T) {
+		gen := ngmodels.RuleGen
+
+		t.Run("when it is the only alert", func(t *testing.T) {
+			fakeStore, fakeAIM, api := setupAPI(t)
+			rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
+			fakeAIM.GenerateAlertInstances(1, rule.UID, 1, withRecoveringState())
+			fakeStore.PutRule(context.Background(), rule)
+
+			r := api.RouteGetRuleStatuses(c)
+			require.Equal(t, http.StatusOK, r.Status())
+
+			var res apimodels.RuleResponse
+			require.NoError(t, json.Unmarshal(r.Body(), &res))
+
+			// There should be 1 recovering rule
+			require.Equal(t, map[string]int64{"recovering": 1}, res.Data.Totals)
+			require.Len(t, res.Data.RuleGroups, 1)
+			rg := res.Data.RuleGroups[0]
+			require.Len(t, rg.Rules, 1)
+			require.Equal(t, "recovering", rg.Rules[0].State)
+
+			// The rule should have one recovering alert
+			require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
+			require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
+			require.Len(t, rg.Rules[0].Alerts, 1)
+			require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
+		})
+
+		t.Run("when the rule has also a firing alert", func(t *testing.T) {
+			fakeStore, fakeAIM, api := setupAPI(t)
+			rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
+			fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withRecoveringState())
+			fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withAlertingState())
+			fakeStore.PutRule(context.Background(), rule)
+
+			r := api.RouteGetRuleStatuses(c)
+			require.Equal(t, http.StatusOK, r.Status())
+
+			var res apimodels.RuleResponse
+			require.NoError(t, json.Unmarshal(r.Body(), &res))
+
+			// There should be 1 firing rule
+			require.Equal(t, map[string]int64{"firing": 1}, res.Data.Totals)
+			require.Len(t, res.Data.RuleGroups, 1)
+			rg := res.Data.RuleGroups[0]
+			require.Len(t, rg.Rules, 1)
+			require.Equal(t, "firing", rg.Rules[0].State)
+
+			// The rule should have one firing and one recovering alert
+			require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].Totals)
+			require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].TotalsFiltered)
+			require.Len(t, rg.Rules[0].Alerts, 2)
+			alertStates := []string{rg.Rules[0].Alerts[0].State, rg.Rules[0].Alerts[1].State}
+			require.ElementsMatch(t, alertStates, []string{"Alerting", "Recovering"})
+		})
+
+		t.Run("filtered by recovering state", func(t *testing.T) {
+			fakeStore, fakeAIM, api := setupAPI(t)
+			groupKey := ngmodels.GenerateGroupKey(orgID)
+			recoveringRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
+			alertingRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
+			fakeAIM.GenerateAlertInstances(orgID, recoveringRule.UID, 1, withRecoveringState())
+			fakeAIM.GenerateAlertInstances(orgID, alertingRule.UID, 1, withAlertingState())
+			fakeStore.PutRule(context.Background(), recoveringRule)
+			fakeStore.PutRule(context.Background(), alertingRule)
+
+			req, err := http.NewRequest("GET", "/api/v1/rules?state=recovering", nil)
+			require.NoError(t, err)
+			c := &contextmodel.ReqContext{
+				Context: &web.Context{Req: req},
+				SignedInUser: &user.SignedInUser{
+					OrgID:       orgID,
+					Permissions: queryPermissions,
+				},
+			}
+			r := api.RouteGetRuleStatuses(c)
+			require.Equal(t, http.StatusOK, r.Status())
+
+			var res apimodels.RuleResponse
+			require.NoError(t, json.Unmarshal(r.Body(), &res))
+
+			// global totals aren't filtered
+			require.Equal(t, map[string]int64{"recovering": 1, "firing": 1}, res.Data.Totals)
+			require.Len(t, res.Data.RuleGroups, 1)
+			rg := res.Data.RuleGroups[0]
+			require.Len(t, rg.Rules, 1)
+			require.Equal(t, "recovering", rg.Rules[0].State)
+
+			// The rule should have one recovering alert
+			require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
+			require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
+			require.Len(t, rg.Rules[0].Alerts, 1)
+			require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
+		})
+	})
+
 	t.Run("with many rules in a group", func(t *testing.T) {
 		t.Run("should return sorted", func(t *testing.T) {
 			ruleStore := fakes.NewRuleStore(t)
@@ -1570,6 +1715,7 @@ func asFixture() ngmodels.AlertRuleMutator {
 		r.Annotations = nil
 		r.IntervalSeconds = 60
 		r.For = 180 * time.Second
+		r.KeepFiringFor = 10 * time.Second
 	}
 }
 

pkg/services/ngalert/api/api_ruler.go

@@ -662,10 +662,12 @@ func toGettableExtendedRuleNode(r ngmodels.AlertRule, provenanceRecords map[stri
 		},
 	}
 	forDuration := model.Duration(r.For)
+	keepFiringForDuration := model.Duration(r.KeepFiringFor)
 	gettableExtendedRuleNode.ApiRuleNode = &apimodels.ApiRuleNode{
-		For:         &forDuration,
-		Annotations: r.Annotations,
-		Labels:      r.Labels,
+		For:           &forDuration,
+		KeepFiringFor: &keepFiringForDuration,
+		Annotations:   r.Annotations,
+		Labels:        r.Labels,
 	}
 	return gettableExtendedRuleNode
 }

pkg/services/ngalert/api/api_ruler_test.go

@@ -340,6 +340,7 @@ func TestRouteGetRuleByUID(t *testing.T) {
 			gen.WithUniqueGroupIndex(), gen.WithUniqueID(),
 			gen.WithEditorSettingsSimplifiedQueryAndExpressionsSection(true),
 			gen.WithEditorSettingsSimplifiedNotificationsSection(true),
+			gen.WithKeepFiringFor(30*time.Second),
 		).GenerateManyRef(3)
 		require.Len(t, createdRules, 3)
 		ruleStore.PutRule(context.Background(), createdRules...)
@@ -358,6 +359,7 @@ func TestRouteGetRuleByUID(t *testing.T) {
 		require.Equal(t, expectedRule.UID, result.GrafanaManagedAlert.UID)
 		require.Equal(t, expectedRule.RuleGroup, result.GrafanaManagedAlert.RuleGroup)
 		require.Equal(t, expectedRule.Title, result.GrafanaManagedAlert.Title)
+		require.Equal(t, int64(expectedRule.KeepFiringFor), int64(*(result.KeepFiringFor)))
 		require.True(t, result.GrafanaManagedAlert.Metadata.EditorSettings.SimplifiedQueryAndExpressionsSection)
 		require.True(t, result.GrafanaManagedAlert.Metadata.EditorSettings.SimplifiedNotificationsSection)
 

pkg/services/ngalert/api/api_ruler_validation_test.go

@@ -58,10 +58,12 @@ func allowRecording(lim RuleLimits) *RuleLimits {
 
 func validRule() apimodels.PostableExtendedRuleNode {
 	forDuration := model.Duration(rand.Int63n(1000))
+	keepFiringForDuration := model.Duration(rand.Int63n(1000))
 	uid := util.GenerateShortUID()
 	return apimodels.PostableExtendedRuleNode{
 		ApiRuleNode: &apimodels.ApiRuleNode{
-			For: &forDuration,
+			For:           &forDuration,
+			KeepFiringFor: &keepFiringForDuration,
 			Labels: map[string]string{
 				"test-label": "data",
 			},
@@ -385,6 +387,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 				require.Equal(t, models.NoDataState(api.GrafanaManagedAlert.NoDataState), alert.NoDataState)
 				require.Equal(t, models.ExecutionErrorState(api.GrafanaManagedAlert.ExecErrState), alert.ExecErrState)
 				require.Equal(t, time.Duration(*api.ApiRuleNode.For), alert.For)
+				require.Equal(t, time.Duration(*api.ApiRuleNode.KeepFiringFor), alert.KeepFiringFor)
 				require.Equal(t, api.ApiRuleNode.Annotations, alert.Annotations)
 				require.Equal(t, api.ApiRuleNode.Labels, alert.Labels)
 				require.Nil(t, alert.Record)
@@ -399,6 +402,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 			},
 			assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
 				require.Equal(t, time.Duration(0), alert.For)
+				require.Equal(t, time.Duration(0), alert.KeepFiringFor)
 				require.Nil(t, alert.Annotations)
 				require.Nil(t, alert.Labels)
 			},
@@ -453,6 +457,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 				r.GrafanaManagedAlert.ExecErrState = ""
 				r.GrafanaManagedAlert.NotificationSettings = nil
 				r.ApiRuleNode.For = nil
+				r.ApiRuleNode.KeepFiringFor = nil
 				return &r
 			},
 			assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -477,6 +482,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 				require.Empty(t, alert.ExecErrState)
 				require.Nil(t, alert.NotificationSettings)
 				require.Zero(t, alert.For)
+				require.Zero(t, alert.KeepFiringFor)
 				// Recording fields
 				require.Equal(t, api.GrafanaManagedAlert.Record.From, alert.Record.From)
 				require.Equal(t, api.GrafanaManagedAlert.Record.Metric, alert.Record.Metric)
@@ -493,6 +499,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 				r.GrafanaManagedAlert.ExecErrState = apimodels.AlertingErrState
 				r.GrafanaManagedAlert.NotificationSettings = &apimodels.AlertRuleNotificationSettings{}
 				r.ApiRuleNode.For = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
+				r.ApiRuleNode.KeepFiringFor = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
 				return &r
 			},
 			assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -501,6 +508,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
 				require.Empty(t, alert.ExecErrState)
 				require.Nil(t, alert.NotificationSettings)
 				require.Zero(t, alert.For)
+				require.Zero(t, alert.KeepFiringFor)
 			},
 		},
 	}
@@ -723,6 +731,15 @@ func TestValidateRuleNodeFailures_NoUID(t *testing.T) {
 			},
 			expErr: "NOTEXIST does not exist",
 		},
+		{
+			name: "fail if keep_firing_for is negative",
+			rule: func() *apimodels.PostableExtendedRuleNode {
+				r := validRule()
+				keepFiringFor := model.Duration(-1)
+				r.KeepFiringFor = &keepFiringFor
+				return &r
+			},
+		},
 	}
 
 	for _, testCase := range testCases {

pkg/services/ngalert/api/compat/compat.go

@@ -29,6 +29,7 @@ func AlertRuleFromProvisionedAlertRule(a definitions.ProvisionedAlertRule) (mode
 		NoDataState:          models.NoDataState(a.NoDataState),          // TODO there must be a validation
 		ExecErrState:         models.ExecutionErrorState(a.ExecErrState), // TODO there must be a validation
 		For:                  time.Duration(a.For),
+		KeepFiringFor:        time.Duration(a.KeepFiringFor),
 		Annotations:          a.Annotations,
 		Labels:               a.Labels,
 		IsPaused:             a.IsPaused,
@@ -53,6 +54,7 @@ func ProvisionedAlertRuleFromAlertRule(rule models.AlertRule, provenance models.
 		RuleGroup:            rule.RuleGroup,
 		Title:                rule.Title,
 		For:                  model.Duration(rule.For),
+		KeepFiringFor:        model.Duration(rule.KeepFiringFor),
 		Condition:            rule.Condition,
 		Data:                 ApiAlertQueriesFromAlertQueries(rule.Data),
 		Updated:              rule.Updated,
@@ -206,6 +208,7 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
 		UID:                  rule.UID,
 		Title:                rule.Title,
 		For:                  model.Duration(rule.For),
+		KeepFiringFor:        model.Duration(rule.KeepFiringFor),
 		Condition:            cPtr,
 		Data:                 data,
 		DashboardUID:         rule.DashboardUID,
@@ -219,6 +222,9 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
 	if rule.For.Seconds() > 0 {
 		result.ForString = util.Pointer(model.Duration(rule.For).String())
 	}
+	if rule.KeepFiringFor.Seconds() > 0 {
+		result.KeepFiringForString = util.Pointer(model.Duration(rule.KeepFiringFor).String())
+	}
 	if rule.Annotations != nil {
 		result.Annotations = &rule.Annotations
 	}

pkg/services/ngalert/api/compat/compat_test.go

@@ -2,7 +2,9 @@ package api
 
 import (
 	"testing"
+	"time"
 
+	prommodel "github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -42,11 +44,12 @@ func TestToModel(t *testing.T) {
 			Interval:  10,
 			Rules: []definitions.ProvisionedAlertRule{
 				{
-					UID:          "1",
-					Condition:    "A",
-					ExecErrState: definitions.ErrorErrState,
-					NoDataState:  definitions.NoData,
-					For:          10,
+					UID:           "1",
+					Condition:     "A",
+					ExecErrState:  definitions.ErrorErrState,
+					NoDataState:   definitions.NoData,
+					For:           10,
+					KeepFiringFor: 20,
 					NotificationSettings: &definitions.AlertRuleNotificationSettings{
 						Receiver: "receiver",
 					},
@@ -63,10 +66,53 @@ func TestToModel(t *testing.T) {
 		rule := tm.Rules[0]
 		require.Empty(t, rule.NoDataState)
 		require.Empty(t, rule.For)
+		require.Empty(t, rule.KeepFiringFor)
 		require.Empty(t, rule.Condition)
 		require.Empty(t, rule.ExecErrState)
 		require.Nil(t, rule.NotificationSettings)
 	})
+
+	t.Run("should copy the fields correctly", func(t *testing.T) {
+		ruleGroup := definitions.AlertRuleGroup{
+			Title:     "123",
+			FolderUID: "456",
+			Interval:  int64(10),
+			Rules: []definitions.ProvisionedAlertRule{
+				{
+					UID:           "1",
+					For:           prommodel.Duration(5 * time.Second),
+					KeepFiringFor: prommodel.Duration(15 * time.Second),
+				},
+			},
+		}
+
+		tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
+		require.NoError(t, err)
+
+		require.Equal(t, "123", tm.Title)
+		require.Equal(t, "456", tm.FolderUID)
+		require.Equal(t, int64(10), tm.Interval)
+		require.Len(t, tm.Rules, 1)
+		require.Equal(t, "1", tm.Rules[0].UID)
+		require.Equal(t, time.Second*5, tm.Rules[0].For)
+		require.Equal(t, time.Second*15, tm.Rules[0].KeepFiringFor)
+	})
+
+	t.Run("should handle empty keep firing for", func(t *testing.T) {
+		ruleGroup := definitions.AlertRuleGroup{
+			Title:     "123",
+			FolderUID: "456",
+			Interval:  int64(10),
+			Rules: []definitions.ProvisionedAlertRule{
+				{
+					UID: "1",
+				},
+			},
+		}
+		tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
+		require.NoError(t, err)
+		require.Equal(t, time.Duration(0), tm.Rules[0].KeepFiringFor)
+	})
 }
 
 func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
@@ -84,3 +130,52 @@ func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
 		require.True(t, apiMetadata.EditorSettings.SimplifiedNotificationsSection)
 	})
 }
+
+func TestApiAlertRuleGroupFromAlertRuleGroup(t *testing.T) {
+	t.Run("should convert keepfiringfor duration correctly", func(t *testing.T) {
+		keepFiringFor := 30 * time.Second
+		modelGroup := models.AlertRuleGroup{
+			Title:     "test_group",
+			FolderUID: "folder123",
+			Interval:  int64(10),
+			Rules: []models.AlertRule{
+				{
+					UID:           "rule1",
+					Title:         "Test Rule",
+					For:           10 * time.Second,
+					KeepFiringFor: keepFiringFor,
+				},
+			},
+		}
+
+		apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
+
+		require.Equal(t, "test_group", apiGroup.Title)
+		require.Equal(t, "folder123", apiGroup.FolderUID)
+		require.Equal(t, int64(10), apiGroup.Interval)
+		require.Len(t, apiGroup.Rules, 1)
+
+		rule := apiGroup.Rules[0]
+		require.Equal(t, "rule1", rule.UID)
+		require.Equal(t, prommodel.Duration(keepFiringFor), rule.KeepFiringFor)
+	})
+
+	t.Run("handles empty keep_firing_for", func(t *testing.T) {
+		modelGroup := models.AlertRuleGroup{
+			Title:     "test_group",
+			FolderUID: "folder123",
+			Interval:  int64(10),
+			Rules: []models.AlertRule{
+				{
+					UID:   "rule1",
+					Title: "Test Rule",
+					For:   10 * time.Second,
+				},
+			},
+		}
+
+		apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
+		require.Len(t, apiGroup.Rules, 1)
+		require.Equal(t, prommodel.Duration(0), apiGroup.Rules[0].KeepFiringFor)
+	})
+}

pkg/services/ngalert/api/prometheus/api_prometheus.go

@@ -119,7 +119,7 @@ func PrepareAlertStatuses(manager state.AlertInstanceManager, opts AlertStatuses
 		startsAt := alertState.StartsAt
 		valString := ""
 
-		if alertState.State == eval.Alerting || alertState.State == eval.Pending {
+		if alertState.State == eval.Alerting || alertState.State == eval.Pending || alertState.State == eval.Recovering {
 			valString = FormatValues(alertState)
 		}
 
@@ -204,6 +204,8 @@ func getStatesFromQuery(v url.Values) ([]eval.State, error) {
 		// nolint:goconst
 		case "error":
 			states = append(states, eval.Error)
+		case "recovering":
+			states = append(states, eval.Recovering)
 		default:
 			return states, fmt.Errorf("unknown state '%s'", s)
 		}
@@ -499,6 +501,8 @@ func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]s
 			state = util.Pointer(eval.Alerting)
 		case "pending":
 			state = util.Pointer(eval.Pending)
+		case "recovering":
+			state = util.Pointer(eval.Recovering)
 		}
 		if state != nil {
 			if _, ok := withStatesFast[*state]; ok {
@@ -541,11 +545,12 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
 		}
 
 		alertingRule := apimodels.AlertingRule{
-			State:       "inactive",
-			Name:        rule.Title,
-			Query:       ruleToQuery(log, rule),
-			Duration:    rule.For.Seconds(),
-			Annotations: apimodels.LabelsFromMap(rule.Annotations),
+			State:         "inactive",
+			Name:          rule.Title,
+			Query:         ruleToQuery(log, rule),
+			Duration:      rule.For.Seconds(),
+			KeepFiringFor: rule.KeepFiringFor.Seconds(),
+			Annotations:   apimodels.LabelsFromMap(rule.Annotations),
 		}
 
 		newRule := apimodels.Rule{
@@ -566,7 +571,7 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
 		for _, alertState := range states {
 			activeAt := alertState.StartsAt
 			valString := ""
-			if alertState.State == eval.Alerting || alertState.State == eval.Pending {
+			if alertState.State == eval.Alerting || alertState.State == eval.Pending || alertState.State == eval.Recovering {
 				valString = FormatValues(alertState)
 			}
 			stateKey := strings.ToLower(alertState.State.String())
@@ -586,12 +591,19 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
 				Value:    valString,
 			}
 
+			// Set the state of the rule based on the state of its alerts.
+			// Only update the rule state with 'pending' or 'recovering' if the current state is 'inactive'.
+			// This prevents overwriting a higher-severity 'firing' state in the case of a rule with multiple alerts.
 			switch alertState.State {
 			case eval.Normal:
 			case eval.Pending:
 				if alertingRule.State == "inactive" {
 					alertingRule.State = "pending"
 				}
+			case eval.Recovering:
+				if alertingRule.State == "inactive" {
+					alertingRule.State = "recovering"
+				}
 			case eval.Alerting:
 				if alertingRule.ActiveAt == nil || alertingRule.ActiveAt.After(activeAt) {
 					alertingRule.ActiveAt = &activeAt

pkg/services/ngalert/api/tooling/definitions/prom.go

@@ -152,8 +152,9 @@ type AlertingRule struct {
 	// required: true
 	Name string `json:"name,omitempty"`
 	// required: true
-	Query    string  `json:"query,omitempty"`
-	Duration float64 `json:"duration,omitempty"`
+	Query         string  `json:"query,omitempty"`
+	Duration      float64 `json:"duration,omitempty"`
+	KeepFiringFor float64 `json:"keepFiringFor,omitempty"`
 	// required: true
 	Annotations promlabels.Labels `json:"annotations,omitempty"`
 	// required: true

pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go

@@ -158,6 +158,9 @@ type ProvisionedAlertRule struct {
 	// required: true
 	// swagger:strfmt duration
 	For model.Duration `json:"for"`
+	// required: false
+	// swagger:strfmt duration
+	KeepFiringFor model.Duration `json:"keep_firing_for"`
 	// example: {"runbook_url": "https://supercoolrunbook.com/page/13"}
 	Annotations map[string]string `json:"annotations,omitempty"`
 	// example: {"team": "sre-team-1"}
@@ -259,19 +262,21 @@ type AlertRuleGroupExport struct {
 
 // AlertRuleExport is the provisioned file export of models.AlertRule.
 type AlertRuleExport struct {
-	UID          string               `json:"uid,omitempty" yaml:"uid,omitempty"`
-	Title        string               `json:"title" yaml:"title" hcl:"name"`
-	Condition    *string              `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
-	Data         []AlertQueryExport   `json:"data" yaml:"data" hcl:"data,block"`
-	DashboardUID *string              `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
-	PanelID      *int64               `json:"panelId,omitempty" yaml:"panelId,omitempty"`
-	NoDataState  *NoDataState         `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
-	ExecErrState *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
-	For          model.Duration       `json:"for,omitempty" yaml:"for,omitempty"`
-	// ForString is used to:
+	UID           string               `json:"uid,omitempty" yaml:"uid,omitempty"`
+	Title         string               `json:"title" yaml:"title" hcl:"name"`
+	Condition     *string              `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
+	Data          []AlertQueryExport   `json:"data" yaml:"data" hcl:"data,block"`
+	DashboardUID  *string              `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
+	PanelID       *int64               `json:"panelId,omitempty" yaml:"panelId,omitempty"`
+	NoDataState   *NoDataState         `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
+	ExecErrState  *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
+	For           model.Duration       `json:"for,omitempty" yaml:"for,omitempty"`
+	KeepFiringFor model.Duration       `json:"keepFiringFor,omitempty" yaml:"keepFiringFor,omitempty" hcl:"keep_firing_for"`
+	// ForString and KeepFiringForString are used to:
 	// - Only export the for field for HCL if it is non-zero.
 	// - Format the Prometheus model.Duration type properly for HCL.
 	ForString            *string                              `json:"-" yaml:"-" hcl:"for"`
+	KeepFiringForString  *string                              `json:"-" yaml:"-" hcl:"keep_firing_for"`
 	Annotations          *map[string]string                   `json:"annotations,omitempty" yaml:"annotations,omitempty" hcl:"annotations"`
 	Labels               *map[string]string                   `json:"labels,omitempty" yaml:"labels,omitempty" hcl:"labels"`
 	IsPaused             bool                                 `json:"isPaused" yaml:"isPaused" hcl:"is_paused"`

pkg/services/ngalert/api/validation/api_ruler_validation.go

@@ -157,6 +157,11 @@ func validateAlertingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
 		return ngmodels.AlertRule{}, err
 	}
 
+	newRule.KeepFiringFor, err = validateKeepFiringForInterval(in)
+	if err != nil {
+		return ngmodels.AlertRule{}, err
+	}
+
 	return newRule, nil
 }
 
@@ -185,6 +190,7 @@ func validateRecordingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
 	newRule.ExecErrState = ""
 	newRule.Condition = ""
 	newRule.For = 0
+	newRule.KeepFiringFor = 0
 	newRule.NotificationSettings = nil
 
 	return newRule, nil
@@ -272,6 +278,21 @@ func validateForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Dur
 	return duration, nil
 }
 
+// validateKeepFiringForInterval validates ApiRuleNode.KeepFiringFor and converts it to time.Duration. If the field is not specified returns 0 if GrafanaManagedAlert.UID is empty and -1 if it is not.
+func validateKeepFiringForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Duration, error) {
+	if ruleNode.ApiRuleNode == nil || ruleNode.ApiRuleNode.KeepFiringFor == nil {
+		if ruleNode.GrafanaManagedAlert.UID != "" {
+			return -1, nil // will be patched later with the real value of the current version of the rule
+		}
+		return 0, nil // if it's a new rule, use the 0 as the default
+	}
+	duration := time.Duration(*ruleNode.ApiRuleNode.KeepFiringFor)
+	if duration < 0 {
+		return 0, fmt.Errorf("field `keep_firing_for` cannot be negative [%v]. only 0 or any positive value is allowed", *ruleNode.ApiRuleNode.KeepFiringFor)
+	}
+	return duration, nil
+}
+
 // ValidateRuleGroup validates API model (definitions.PostableRuleGroupConfig) and converts it to a collection of models.AlertRule.
 // Returns a slice that contains all rules described by API model or error if either group specification or an alert definition is not valid.
 // It also returns a map containing current existing alerts that don't contain the is_paused field in the body of the call.