public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add backend support for keep_firing_for (#100750)

Browse Source 
What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert
This commit is contained in:
Alexander Akhmetov
2025-03-18 11:24:48 +01:00
committed by GitHub
parent 9491fa1895
commit 695ac91290
31 changed files with 1280 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+146
View File
@@ -191,6 +191,38 @@ func TestRouteGetAlertStatuses(t *testing.T) {
}`, string(r.Body()))
})
t.Run("with a recovering alert", func(t *testing.T) {
_, fakeAIM, api := setupAPI(t)
fakeAIM.GenerateAlertInstances(1, util.GenerateShortUID(), 1, withRecoveringState())
req, err := http.NewRequest("GET", "/api/v1/alerts", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{Context: &web.Context{Req: req}, SignedInUser: &user.SignedInUser{OrgID: orgID}}
r := api.RouteGetAlertStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
require.JSONEq(t, `
{
"status": "success",
"data": {
"alerts": [{
"labels": {
"alertname": "test_title_0",
"instance_label": "test",
"label": "test"
},
"annotations": {
"annotation": "test"
},
"state": "Recovering",
"activeAt": "0001-01-01T00:00:00Z",
"value": "1.1e+00"
}]
}
}`,
string(r.Body()),
)
})
t.Run("with the inclusion of internal labels", func(t *testing.T) {
_, fakeAIM, api := setupAPI(t)
fakeAIM.GenerateAlertInstances(orgID, util.GenerateShortUID(), 2)
@@ -251,6 +283,19 @@ func withAlertingState() forEachState {
}
}
func withRecoveringState() forEachState {
return func(s *state.State) *state.State {
s.State = eval.Recovering
s.LatestResult = &state.Evaluation{
EvaluationState: eval.Alerting,
EvaluationTime: timeNow(),
Values: map[string]float64{"B": float64(1.1)},
Condition: "B",
}
return s
}
}
func withAlertingErrorState() forEachState {
return func(s *state.State) *state.State {
s.SetAlerting("", timeNow(), timeNow().Add(5*time.Minute))
@@ -347,6 +392,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -416,6 +462,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -478,6 +525,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -495,6 +543,103 @@ func TestRouteGetRuleStatuses(t *testing.T) {
`, folder.Fullpath), string(r.Body()))
})
t.Run("with a recovering alert", func(t *testing.T) {
gen := ngmodels.RuleGen
t.Run("when it is the only alert", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(1, rule.UID, 1, withRecoveringState())
fakeStore.PutRule(context.Background(), rule)
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// There should be 1 recovering rule
require.Equal(t, map[string]int64{"recovering": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "recovering", rg.Rules[0].State)
// The rule should have one recovering alert
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 1)
require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
})
t.Run("when the rule has also a firing alert", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withRecoveringState())
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withAlertingState())
fakeStore.PutRule(context.Background(), rule)
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// There should be 1 firing rule
require.Equal(t, map[string]int64{"firing": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "firing", rg.Rules[0].State)
// The rule should have one firing and one recovering alert
require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 2)
alertStates := []string{rg.Rules[0].Alerts[0].State, rg.Rules[0].Alerts[1].State}
require.ElementsMatch(t, alertStates, []string{"Alerting", "Recovering"})
})
t.Run("filtered by recovering state", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
groupKey := ngmodels.GenerateGroupKey(orgID)
recoveringRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
alertingRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, recoveringRule.UID, 1, withRecoveringState())
fakeAIM.GenerateAlertInstances(orgID, alertingRule.UID, 1, withAlertingState())
fakeStore.PutRule(context.Background(), recoveringRule)
fakeStore.PutRule(context.Background(), alertingRule)
req, err := http.NewRequest("GET", "/api/v1/rules?state=recovering", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: req},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// global totals aren't filtered
require.Equal(t, map[string]int64{"recovering": 1, "firing": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "recovering", rg.Rules[0].State)
// The rule should have one recovering alert
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 1)
require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
})
})
t.Run("with many rules in a group", func(t *testing.T) {
t.Run("should return sorted", func(t *testing.T) {
ruleStore := fakes.NewRuleStore(t)
@@ -1570,6 +1715,7 @@ func asFixture() ngmodels.AlertRuleMutator {
r.Annotations = nil
r.IntervalSeconds = 60
r.For = 180 * time.Second
r.KeepFiringFor = 10 * time.Second
}
}