public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add backend support for keep_firing_for (#100750)

Browse Source 
What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert
This commit is contained in:
Alexander Akhmetov
2025-03-18 11:24:48 +01:00
committed by GitHub
parent 9491fa1895
commit 695ac91290
31 changed files with 1280 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/compat/compat.go
+6
View File
@@ -29,6 +29,7 @@ func AlertRuleFromProvisionedAlertRule(a definitions.ProvisionedAlertRule) (mode
NoDataState: models.NoDataState(a.NoDataState), // TODO there must be a validation
ExecErrState: models.ExecutionErrorState(a.ExecErrState), // TODO there must be a validation
For: time.Duration(a.For),
KeepFiringFor: time.Duration(a.KeepFiringFor),
Annotations: a.Annotations,
Labels: a.Labels,
IsPaused: a.IsPaused,
@@ -53,6 +54,7 @@ func ProvisionedAlertRuleFromAlertRule(rule models.AlertRule, provenance models.
RuleGroup: rule.RuleGroup,
Title: rule.Title,
For: model.Duration(rule.For),
KeepFiringFor: model.Duration(rule.KeepFiringFor),
Condition: rule.Condition,
Data: ApiAlertQueriesFromAlertQueries(rule.Data),
Updated: rule.Updated,
@@ -206,6 +208,7 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
UID: rule.UID,
Title: rule.Title,
For: model.Duration(rule.For),
KeepFiringFor: model.Duration(rule.KeepFiringFor),
Condition: cPtr,
Data: data,
DashboardUID: rule.DashboardUID,
@@ -219,6 +222,9 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
if rule.For.Seconds() > 0 {
result.ForString = util.Pointer(model.Duration(rule.For).String())
}
if rule.KeepFiringFor.Seconds() > 0 {
result.KeepFiringForString = util.Pointer(model.Duration(rule.KeepFiringFor).String())
}
if rule.Annotations != nil {
result.Annotations = &rule.Annotations
}
pkg/services/ngalert/api/compat/compat_test.go
+100 -5
View File
@@ -2,7 +2,9 @@ package api
import (
"testing"
"time"
prommodel "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -42,11 +44,12 @@ func TestToModel(t *testing.T) {
Interval: 10,
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
Condition: "A",
ExecErrState: definitions.ErrorErrState,
NoDataState: definitions.NoData,
For: 10,
UID: "1",
Condition: "A",
ExecErrState: definitions.ErrorErrState,
NoDataState: definitions.NoData,
For: 10,
KeepFiringFor: 20,
NotificationSettings: &definitions.AlertRuleNotificationSettings{
Receiver: "receiver",
},
@@ -63,10 +66,53 @@ func TestToModel(t *testing.T) {
rule := tm.Rules[0]
require.Empty(t, rule.NoDataState)
require.Empty(t, rule.For)
require.Empty(t, rule.KeepFiringFor)
require.Empty(t, rule.Condition)
require.Empty(t, rule.ExecErrState)
require.Nil(t, rule.NotificationSettings)
})
t.Run("should copy the fields correctly", func(t *testing.T) {
ruleGroup := definitions.AlertRuleGroup{
Title: "123",
FolderUID: "456",
Interval: int64(10),
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
For: prommodel.Duration(5 * time.Second),
KeepFiringFor: prommodel.Duration(15 * time.Second),
},
},
}
tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
require.NoError(t, err)
require.Equal(t, "123", tm.Title)
require.Equal(t, "456", tm.FolderUID)
require.Equal(t, int64(10), tm.Interval)
require.Len(t, tm.Rules, 1)
require.Equal(t, "1", tm.Rules[0].UID)
require.Equal(t, time.Second*5, tm.Rules[0].For)
require.Equal(t, time.Second*15, tm.Rules[0].KeepFiringFor)
})
t.Run("should handle empty keep firing for", func(t *testing.T) {
ruleGroup := definitions.AlertRuleGroup{
Title: "123",
FolderUID: "456",
Interval: int64(10),
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
},
},
}
tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
require.NoError(t, err)
require.Equal(t, time.Duration(0), tm.Rules[0].KeepFiringFor)
})
}
func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
@@ -84,3 +130,52 @@ func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
require.True(t, apiMetadata.EditorSettings.SimplifiedNotificationsSection)
})
}
func TestApiAlertRuleGroupFromAlertRuleGroup(t *testing.T) {
t.Run("should convert keepfiringfor duration correctly", func(t *testing.T) {
keepFiringFor := 30 * time.Second
modelGroup := models.AlertRuleGroup{
Title: "test_group",
FolderUID: "folder123",
Interval: int64(10),
Rules: []models.AlertRule{
{
UID: "rule1",
Title: "Test Rule",
For: 10 * time.Second,
KeepFiringFor: keepFiringFor,
},
},
}
apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
require.Equal(t, "test_group", apiGroup.Title)
require.Equal(t, "folder123", apiGroup.FolderUID)
require.Equal(t, int64(10), apiGroup.Interval)
require.Len(t, apiGroup.Rules, 1)
rule := apiGroup.Rules[0]
require.Equal(t, "rule1", rule.UID)
require.Equal(t, prommodel.Duration(keepFiringFor), rule.KeepFiringFor)
})
t.Run("handles empty keep_firing_for", func(t *testing.T) {
modelGroup := models.AlertRuleGroup{
Title: "test_group",
FolderUID: "folder123",
Interval: int64(10),
Rules: []models.AlertRule{
{
UID: "rule1",
Title: "Test Rule",
For: 10 * time.Second,
},
},
}
apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
require.Len(t, apiGroup.Rules, 1)
require.Equal(t, prommodel.Duration(0), apiGroup.Rules[0].KeepFiringFor)
})
}