public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add backend support for keep_firing_for (#100750)

Browse Source 
What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert
This commit is contained in:
Alexander Akhmetov
2025-03-18 11:24:48 +01:00
committed by GitHub
parent 9491fa1895
commit 695ac91290
31 changed files with 1280 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_ruler_validation_test.go
+18 -1
View File
@@ -58,10 +58,12 @@ func allowRecording(lim RuleLimits) *RuleLimits {
func validRule() apimodels.PostableExtendedRuleNode {
forDuration := model.Duration(rand.Int63n(1000))
keepFiringForDuration := model.Duration(rand.Int63n(1000))
uid := util.GenerateShortUID()
return apimodels.PostableExtendedRuleNode{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &forDuration,
For: &forDuration,
KeepFiringFor: &keepFiringForDuration,
Labels: map[string]string{
"test-label": "data",
},
@@ -385,6 +387,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Equal(t, models.NoDataState(api.GrafanaManagedAlert.NoDataState), alert.NoDataState)
require.Equal(t, models.ExecutionErrorState(api.GrafanaManagedAlert.ExecErrState), alert.ExecErrState)
require.Equal(t, time.Duration(*api.ApiRuleNode.For), alert.For)
require.Equal(t, time.Duration(*api.ApiRuleNode.KeepFiringFor), alert.KeepFiringFor)
require.Equal(t, api.ApiRuleNode.Annotations, alert.Annotations)
require.Equal(t, api.ApiRuleNode.Labels, alert.Labels)
require.Nil(t, alert.Record)
@@ -399,6 +402,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
require.Equal(t, time.Duration(0), alert.For)
require.Equal(t, time.Duration(0), alert.KeepFiringFor)
require.Nil(t, alert.Annotations)
require.Nil(t, alert.Labels)
},
@@ -453,6 +457,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
r.GrafanaManagedAlert.ExecErrState = ""
r.GrafanaManagedAlert.NotificationSettings = nil
r.ApiRuleNode.For = nil
r.ApiRuleNode.KeepFiringFor = nil
return &r
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -477,6 +482,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Empty(t, alert.ExecErrState)
require.Nil(t, alert.NotificationSettings)
require.Zero(t, alert.For)
require.Zero(t, alert.KeepFiringFor)
// Recording fields
require.Equal(t, api.GrafanaManagedAlert.Record.From, alert.Record.From)
require.Equal(t, api.GrafanaManagedAlert.Record.Metric, alert.Record.Metric)
@@ -493,6 +499,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
r.GrafanaManagedAlert.ExecErrState = apimodels.AlertingErrState
r.GrafanaManagedAlert.NotificationSettings = &apimodels.AlertRuleNotificationSettings{}
r.ApiRuleNode.For = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
r.ApiRuleNode.KeepFiringFor = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
return &r
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -501,6 +508,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Empty(t, alert.ExecErrState)
require.Nil(t, alert.NotificationSettings)
require.Zero(t, alert.For)
require.Zero(t, alert.KeepFiringFor)
},
},
}
@@ -723,6 +731,15 @@ func TestValidateRuleNodeFailures_NoUID(t *testing.T) {
},
expErr: "NOTEXIST does not exist",
},
{
name: "fail if keep_firing_for is negative",
rule: func() *apimodels.PostableExtendedRuleNode {
r := validRule()
keepFiringFor := model.Duration(-1)
r.KeepFiringFor = &keepFiringFor
return &r
},
},
}
for _, testCase := range testCases {