public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add backend support for keep_firing_for (#100750)

Browse Source 
What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert
This commit is contained in:
Alexander Akhmetov
2025-03-18 11:24:48 +01:00
committed by GitHub
parent 9491fa1895
commit 695ac91290
31 changed files with 1280 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/tooling/definitions/prom.go
+3 -2
View File
@@ -152,8 +152,9 @@ type AlertingRule struct {
// required: true
Name string `json:"name,omitempty"`
// required: true
Query string `json:"query,omitempty"`
Duration float64 `json:"duration,omitempty"`
Query string `json:"query,omitempty"`
Duration float64 `json:"duration,omitempty"`
KeepFiringFor float64 `json:"keepFiringFor,omitempty"`
// required: true
Annotations promlabels.Labels `json:"annotations,omitempty"`
// required: true
pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go
+15 -10
View File
@@ -158,6 +158,9 @@ type ProvisionedAlertRule struct {
// required: true
// swagger:strfmt duration
For model.Duration `json:"for"`
// required: false
// swagger:strfmt duration
KeepFiringFor model.Duration `json:"keep_firing_for"`
// example: {"runbook_url": "https://supercoolrunbook.com/page/13"}
Annotations map[string]string `json:"annotations,omitempty"`
// example: {"team": "sre-team-1"}
@@ -259,19 +262,21 @@ type AlertRuleGroupExport struct {
// AlertRuleExport is the provisioned file export of models.AlertRule.
type AlertRuleExport struct {
UID string `json:"uid,omitempty" yaml:"uid,omitempty"`
Title string `json:"title" yaml:"title" hcl:"name"`
Condition *string `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
Data []AlertQueryExport `json:"data" yaml:"data" hcl:"data,block"`
DashboardUID *string `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
PanelID *int64 `json:"panelId,omitempty" yaml:"panelId,omitempty"`
NoDataState *NoDataState `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
ExecErrState *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
For model.Duration `json:"for,omitempty" yaml:"for,omitempty"`
// ForString is used to:
UID string `json:"uid,omitempty" yaml:"uid,omitempty"`
Title string `json:"title" yaml:"title" hcl:"name"`
Condition *string `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
Data []AlertQueryExport `json:"data" yaml:"data" hcl:"data,block"`
DashboardUID *string `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
PanelID *int64 `json:"panelId,omitempty" yaml:"panelId,omitempty"`
NoDataState *NoDataState `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
ExecErrState *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
For model.Duration `json:"for,omitempty" yaml:"for,omitempty"`
KeepFiringFor model.Duration `json:"keepFiringFor,omitempty" yaml:"keepFiringFor,omitempty" hcl:"keep_firing_for"`
// ForString and KeepFiringForString are used to:
// - Only export the for field for HCL if it is non-zero.
// - Format the Prometheus model.Duration type properly for HCL.
ForString *string `json:"-" yaml:"-" hcl:"for"`
KeepFiringForString *string `json:"-" yaml:"-" hcl:"keep_firing_for"`
Annotations *map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty" hcl:"annotations"`
Labels *map[string]string `json:"labels,omitempty" yaml:"labels,omitempty" hcl:"labels"`
IsPaused bool `json:"isPaused" yaml:"isPaused" hcl:"is_paused"`