public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add backend support for keep_firing_for (#100750)

Browse Source 
What is this feature?

This PR introduces a new alert rule configuration option, keep_firing_for (Prometheus documentation).

keep_firing_for prevents alerts from resolving immediately after the alert condition returns to normal. Instead, they transition into a "Recovering" state and are not considered resolved by the Alertmanager. Once the recovery period ends (or after the next evaluation if it is bigger than keep_firing_for), the alert transitions to "Normal" if it doesn't start alerting again:

Before                                          

+----------+     +----------+                    
| Alerting |---->|  Normal  |                    
+----------+     +----------+                    

-----
After

+----------+      +------------+     +----------+
| Alerting |----->| Recovering |---->|  Normal  |
+----------+      +------------+     +----------+                                                 

Why do we need this feature?

This feature prevents flapping alerts by adding a recovery period. This helps avoid false resolutions caused by brief alert
This commit is contained in:
Alexander Akhmetov
2025-03-18 11:24:48 +01:00
committed by GitHub
parent 9491fa1895
commit 695ac91290
31 changed files with 1280 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+146
View File
@@ -191,6 +191,38 @@ func TestRouteGetAlertStatuses(t *testing.T) {
}`, string(r.Body()))
})
t.Run("with a recovering alert", func(t *testing.T) {
_, fakeAIM, api := setupAPI(t)
fakeAIM.GenerateAlertInstances(1, util.GenerateShortUID(), 1, withRecoveringState())
req, err := http.NewRequest("GET", "/api/v1/alerts", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{Context: &web.Context{Req: req}, SignedInUser: &user.SignedInUser{OrgID: orgID}}
r := api.RouteGetAlertStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
require.JSONEq(t, `
{
"status": "success",
"data": {
"alerts": [{
"labels": {
"alertname": "test_title_0",
"instance_label": "test",
"label": "test"
},
"annotations": {
"annotation": "test"
},
"state": "Recovering",
"activeAt": "0001-01-01T00:00:00Z",
"value": "1.1e+00"
}]
}
}`,
string(r.Body()),
)
})
t.Run("with the inclusion of internal labels", func(t *testing.T) {
_, fakeAIM, api := setupAPI(t)
fakeAIM.GenerateAlertInstances(orgID, util.GenerateShortUID(), 2)
@@ -251,6 +283,19 @@ func withAlertingState() forEachState {
}
}
func withRecoveringState() forEachState {
return func(s *state.State) *state.State {
s.State = eval.Recovering
s.LatestResult = &state.Evaluation{
EvaluationState: eval.Alerting,
EvaluationTime: timeNow(),
Values: map[string]float64{"B": float64(1.1)},
Condition: "B",
}
return s
}
}
func withAlertingErrorState() forEachState {
return func(s *state.State) *state.State {
s.SetAlerting("", timeNow(), timeNow().Add(5*time.Minute))
@@ -347,6 +392,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -416,6 +462,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -478,6 +525,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"type": "alerting",
"lastEvaluation": "2022-03-10T14:01:00Z",
"duration": 180,
"keepFiringFor": 10,
"evaluationTime": 60
}],
"totals": {
@@ -495,6 +543,103 @@ func TestRouteGetRuleStatuses(t *testing.T) {
`, folder.Fullpath), string(r.Body()))
})
t.Run("with a recovering alert", func(t *testing.T) {
gen := ngmodels.RuleGen
t.Run("when it is the only alert", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(1, rule.UID, 1, withRecoveringState())
fakeStore.PutRule(context.Background(), rule)
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// There should be 1 recovering rule
require.Equal(t, map[string]int64{"recovering": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "recovering", rg.Rules[0].State)
// The rule should have one recovering alert
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 1)
require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
})
t.Run("when the rule has also a firing alert", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rule := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withRecoveringState())
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, withAlertingState())
fakeStore.PutRule(context.Background(), rule)
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// There should be 1 firing rule
require.Equal(t, map[string]int64{"firing": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "firing", rg.Rules[0].State)
// The rule should have one firing and one recovering alert
require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"alerting": 1, "recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 2)
alertStates := []string{rg.Rules[0].Alerts[0].State, rg.Rules[0].Alerts[1].State}
require.ElementsMatch(t, alertStates, []string{"Alerting", "Recovering"})
})
t.Run("filtered by recovering state", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
groupKey := ngmodels.GenerateGroupKey(orgID)
recoveringRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
alertingRule := gen.With(gen.WithOrgID(orgID), gen.WithGroupKey(groupKey), withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, recoveringRule.UID, 1, withRecoveringState())
fakeAIM.GenerateAlertInstances(orgID, alertingRule.UID, 1, withAlertingState())
fakeStore.PutRule(context.Background(), recoveringRule)
fakeStore.PutRule(context.Background(), alertingRule)
req, err := http.NewRequest("GET", "/api/v1/rules?state=recovering", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: req},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
r := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, r.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(r.Body(), &res))
// global totals aren't filtered
require.Equal(t, map[string]int64{"recovering": 1, "firing": 1}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "recovering", rg.Rules[0].State)
// The rule should have one recovering alert
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].Totals)
require.Equal(t, map[string]int64{"recovering": 1}, rg.Rules[0].TotalsFiltered)
require.Len(t, rg.Rules[0].Alerts, 1)
require.Equal(t, "Recovering", rg.Rules[0].Alerts[0].State)
})
})
t.Run("with many rules in a group", func(t *testing.T) {
t.Run("should return sorted", func(t *testing.T) {
ruleStore := fakes.NewRuleStore(t)
@@ -1570,6 +1715,7 @@ func asFixture() ngmodels.AlertRuleMutator {
r.Annotations = nil
r.IntervalSeconds = 60
r.For = 180 * time.Second
r.KeepFiringFor = 10 * time.Second
}
}
pkg/services/ngalert/api/api_ruler.go
+5 -3
View File
@@ -662,10 +662,12 @@ func toGettableExtendedRuleNode(r ngmodels.AlertRule, provenanceRecords map[stri
},
}
forDuration := model.Duration(r.For)
keepFiringForDuration := model.Duration(r.KeepFiringFor)
gettableExtendedRuleNode.ApiRuleNode = &apimodels.ApiRuleNode{
For: &forDuration,
Annotations: r.Annotations,
Labels: r.Labels,
For: &forDuration,
KeepFiringFor: &keepFiringForDuration,
Annotations: r.Annotations,
Labels: r.Labels,
}
return gettableExtendedRuleNode
}
pkg/services/ngalert/api/api_ruler_test.go
+2
View File
@@ -340,6 +340,7 @@ func TestRouteGetRuleByUID(t *testing.T) {
gen.WithUniqueGroupIndex(), gen.WithUniqueID(),
gen.WithEditorSettingsSimplifiedQueryAndExpressionsSection(true),
gen.WithEditorSettingsSimplifiedNotificationsSection(true),
gen.WithKeepFiringFor(30*time.Second),
).GenerateManyRef(3)
require.Len(t, createdRules, 3)
ruleStore.PutRule(context.Background(), createdRules...)
@@ -358,6 +359,7 @@ func TestRouteGetRuleByUID(t *testing.T) {
require.Equal(t, expectedRule.UID, result.GrafanaManagedAlert.UID)
require.Equal(t, expectedRule.RuleGroup, result.GrafanaManagedAlert.RuleGroup)
require.Equal(t, expectedRule.Title, result.GrafanaManagedAlert.Title)
require.Equal(t, int64(expectedRule.KeepFiringFor), int64(*(result.KeepFiringFor)))
require.True(t, result.GrafanaManagedAlert.Metadata.EditorSettings.SimplifiedQueryAndExpressionsSection)
require.True(t, result.GrafanaManagedAlert.Metadata.EditorSettings.SimplifiedNotificationsSection)
pkg/services/ngalert/api/api_ruler_validation_test.go
+18 -1
View File
@@ -58,10 +58,12 @@ func allowRecording(lim RuleLimits) *RuleLimits {
func validRule() apimodels.PostableExtendedRuleNode {
forDuration := model.Duration(rand.Int63n(1000))
keepFiringForDuration := model.Duration(rand.Int63n(1000))
uid := util.GenerateShortUID()
return apimodels.PostableExtendedRuleNode{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &forDuration,
For: &forDuration,
KeepFiringFor: &keepFiringForDuration,
Labels: map[string]string{
"test-label": "data",
},
@@ -385,6 +387,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Equal(t, models.NoDataState(api.GrafanaManagedAlert.NoDataState), alert.NoDataState)
require.Equal(t, models.ExecutionErrorState(api.GrafanaManagedAlert.ExecErrState), alert.ExecErrState)
require.Equal(t, time.Duration(*api.ApiRuleNode.For), alert.For)
require.Equal(t, time.Duration(*api.ApiRuleNode.KeepFiringFor), alert.KeepFiringFor)
require.Equal(t, api.ApiRuleNode.Annotations, alert.Annotations)
require.Equal(t, api.ApiRuleNode.Labels, alert.Labels)
require.Nil(t, alert.Record)
@@ -399,6 +402,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
require.Equal(t, time.Duration(0), alert.For)
require.Equal(t, time.Duration(0), alert.KeepFiringFor)
require.Nil(t, alert.Annotations)
require.Nil(t, alert.Labels)
},
@@ -453,6 +457,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
r.GrafanaManagedAlert.ExecErrState = ""
r.GrafanaManagedAlert.NotificationSettings = nil
r.ApiRuleNode.For = nil
r.ApiRuleNode.KeepFiringFor = nil
return &r
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -477,6 +482,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Empty(t, alert.ExecErrState)
require.Nil(t, alert.NotificationSettings)
require.Zero(t, alert.For)
require.Zero(t, alert.KeepFiringFor)
// Recording fields
require.Equal(t, api.GrafanaManagedAlert.Record.From, alert.Record.From)
require.Equal(t, api.GrafanaManagedAlert.Record.Metric, alert.Record.Metric)
@@ -493,6 +499,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
r.GrafanaManagedAlert.ExecErrState = apimodels.AlertingErrState
r.GrafanaManagedAlert.NotificationSettings = &apimodels.AlertRuleNotificationSettings{}
r.ApiRuleNode.For = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
r.ApiRuleNode.KeepFiringFor = func() *model.Duration { five := model.Duration(time.Second * 5); return &five }()
return &r
},
assert: func(t *testing.T, api *apimodels.PostableExtendedRuleNode, alert *models.AlertRule) {
@@ -501,6 +508,7 @@ func TestValidateRuleNode_NoUID(t *testing.T) {
require.Empty(t, alert.ExecErrState)
require.Nil(t, alert.NotificationSettings)
require.Zero(t, alert.For)
require.Zero(t, alert.KeepFiringFor)
},
},
}
@@ -723,6 +731,15 @@ func TestValidateRuleNodeFailures_NoUID(t *testing.T) {
},
expErr: "NOTEXIST does not exist",
},
{
name: "fail if keep_firing_for is negative",
rule: func() *apimodels.PostableExtendedRuleNode {
r := validRule()
keepFiringFor := model.Duration(-1)
r.KeepFiringFor = &keepFiringFor
return &r
},
},
}
for _, testCase := range testCases {
pkg/services/ngalert/api/compat/compat.go
+6
View File
@@ -29,6 +29,7 @@ func AlertRuleFromProvisionedAlertRule(a definitions.ProvisionedAlertRule) (mode
NoDataState: models.NoDataState(a.NoDataState), // TODO there must be a validation
ExecErrState: models.ExecutionErrorState(a.ExecErrState), // TODO there must be a validation
For: time.Duration(a.For),
KeepFiringFor: time.Duration(a.KeepFiringFor),
Annotations: a.Annotations,
Labels: a.Labels,
IsPaused: a.IsPaused,
@@ -53,6 +54,7 @@ func ProvisionedAlertRuleFromAlertRule(rule models.AlertRule, provenance models.
RuleGroup: rule.RuleGroup,
Title: rule.Title,
For: model.Duration(rule.For),
KeepFiringFor: model.Duration(rule.KeepFiringFor),
Condition: rule.Condition,
Data: ApiAlertQueriesFromAlertQueries(rule.Data),
Updated: rule.Updated,
@@ -206,6 +208,7 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
UID: rule.UID,
Title: rule.Title,
For: model.Duration(rule.For),
KeepFiringFor: model.Duration(rule.KeepFiringFor),
Condition: cPtr,
Data: data,
DashboardUID: rule.DashboardUID,
@@ -219,6 +222,9 @@ func AlertRuleExportFromAlertRule(rule models.AlertRule) (definitions.AlertRuleE
if rule.For.Seconds() > 0 {
result.ForString = util.Pointer(model.Duration(rule.For).String())
}
if rule.KeepFiringFor.Seconds() > 0 {
result.KeepFiringForString = util.Pointer(model.Duration(rule.KeepFiringFor).String())
}
if rule.Annotations != nil {
result.Annotations = &rule.Annotations
}
pkg/services/ngalert/api/compat/compat_test.go
+100 -5
View File
@@ -2,7 +2,9 @@ package api
import (
"testing"
"time"
prommodel "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -42,11 +44,12 @@ func TestToModel(t *testing.T) {
Interval: 10,
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
Condition: "A",
ExecErrState: definitions.ErrorErrState,
NoDataState: definitions.NoData,
For: 10,
UID: "1",
Condition: "A",
ExecErrState: definitions.ErrorErrState,
NoDataState: definitions.NoData,
For: 10,
KeepFiringFor: 20,
NotificationSettings: &definitions.AlertRuleNotificationSettings{
Receiver: "receiver",
},
@@ -63,10 +66,53 @@ func TestToModel(t *testing.T) {
rule := tm.Rules[0]
require.Empty(t, rule.NoDataState)
require.Empty(t, rule.For)
require.Empty(t, rule.KeepFiringFor)
require.Empty(t, rule.Condition)
require.Empty(t, rule.ExecErrState)
require.Nil(t, rule.NotificationSettings)
})
t.Run("should copy the fields correctly", func(t *testing.T) {
ruleGroup := definitions.AlertRuleGroup{
Title: "123",
FolderUID: "456",
Interval: int64(10),
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
For: prommodel.Duration(5 * time.Second),
KeepFiringFor: prommodel.Duration(15 * time.Second),
},
},
}
tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
require.NoError(t, err)
require.Equal(t, "123", tm.Title)
require.Equal(t, "456", tm.FolderUID)
require.Equal(t, int64(10), tm.Interval)
require.Len(t, tm.Rules, 1)
require.Equal(t, "1", tm.Rules[0].UID)
require.Equal(t, time.Second*5, tm.Rules[0].For)
require.Equal(t, time.Second*15, tm.Rules[0].KeepFiringFor)
})
t.Run("should handle empty keep firing for", func(t *testing.T) {
ruleGroup := definitions.AlertRuleGroup{
Title: "123",
FolderUID: "456",
Interval: int64(10),
Rules: []definitions.ProvisionedAlertRule{
{
UID: "1",
},
},
}
tm, err := AlertRuleGroupFromApiAlertRuleGroup(ruleGroup)
require.NoError(t, err)
require.Equal(t, time.Duration(0), tm.Rules[0].KeepFiringFor)
})
}
func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
@@ -84,3 +130,52 @@ func TestAlertRuleMetadataFromModelMetadata(t *testing.T) {
require.True(t, apiMetadata.EditorSettings.SimplifiedNotificationsSection)
})
}
func TestApiAlertRuleGroupFromAlertRuleGroup(t *testing.T) {
t.Run("should convert keepfiringfor duration correctly", func(t *testing.T) {
keepFiringFor := 30 * time.Second
modelGroup := models.AlertRuleGroup{
Title: "test_group",
FolderUID: "folder123",
Interval: int64(10),
Rules: []models.AlertRule{
{
UID: "rule1",
Title: "Test Rule",
For: 10 * time.Second,
KeepFiringFor: keepFiringFor,
},
},
}
apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
require.Equal(t, "test_group", apiGroup.Title)
require.Equal(t, "folder123", apiGroup.FolderUID)
require.Equal(t, int64(10), apiGroup.Interval)
require.Len(t, apiGroup.Rules, 1)
rule := apiGroup.Rules[0]
require.Equal(t, "rule1", rule.UID)
require.Equal(t, prommodel.Duration(keepFiringFor), rule.KeepFiringFor)
})
t.Run("handles empty keep_firing_for", func(t *testing.T) {
modelGroup := models.AlertRuleGroup{
Title: "test_group",
FolderUID: "folder123",
Interval: int64(10),
Rules: []models.AlertRule{
{
UID: "rule1",
Title: "Test Rule",
For: 10 * time.Second,
},
},
}
apiGroup := ApiAlertRuleGroupFromAlertRuleGroup(modelGroup)
require.Len(t, apiGroup.Rules, 1)
require.Equal(t, prommodel.Duration(0), apiGroup.Rules[0].KeepFiringFor)
})
}
pkg/services/ngalert/api/prometheus/api_prometheus.go
+19 -7
View File
@@ -119,7 +119,7 @@ func PrepareAlertStatuses(manager state.AlertInstanceManager, opts AlertStatuses
startsAt := alertState.StartsAt
valString := ""
if alertState.State == eval.Alerting || alertState.State == eval.Pending {
if alertState.State == eval.Alerting || alertState.State == eval.Pending || alertState.State == eval.Recovering {
valString = FormatValues(alertState)
}
@@ -204,6 +204,8 @@ func getStatesFromQuery(v url.Values) ([]eval.State, error) {
// nolint:goconst
case "error":
states = append(states, eval.Error)
case "recovering":
states = append(states, eval.Recovering)
default:
return states, fmt.Errorf("unknown state '%s'", s)
}
@@ -499,6 +501,8 @@ func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]s
state = util.Pointer(eval.Alerting)
case "pending":
state = util.Pointer(eval.Pending)
case "recovering":
state = util.Pointer(eval.Recovering)
}
if state != nil {
if _, ok := withStatesFast[*state]; ok {
@@ -541,11 +545,12 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
}
alertingRule := apimodels.AlertingRule{
State: "inactive",
Name: rule.Title,
Query: ruleToQuery(log, rule),
Duration: rule.For.Seconds(),
Annotations: apimodels.LabelsFromMap(rule.Annotations),
State: "inactive",
Name: rule.Title,
Query: ruleToQuery(log, rule),
Duration: rule.For.Seconds(),
KeepFiringFor: rule.KeepFiringFor.Seconds(),
Annotations: apimodels.LabelsFromMap(rule.Annotations),
}
newRule := apimodels.Rule{
@@ -566,7 +571,7 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
for _, alertState := range states {
activeAt := alertState.StartsAt
valString := ""
if alertState.State == eval.Alerting || alertState.State == eval.Pending {
if alertState.State == eval.Alerting || alertState.State == eval.Pending || alertState.State == eval.Recovering {
valString = FormatValues(alertState)
}
stateKey := strings.ToLower(alertState.State.String())
@@ -586,12 +591,19 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
Value: valString,
}
// Set the state of the rule based on the state of its alerts.
// Only update the rule state with 'pending' or 'recovering' if the current state is 'inactive'.
// This prevents overwriting a higher-severity 'firing' state in the case of a rule with multiple alerts.
switch alertState.State {
case eval.Normal:
case eval.Pending:
if alertingRule.State == "inactive" {
alertingRule.State = "pending"
}
case eval.Recovering:
if alertingRule.State == "inactive" {
alertingRule.State = "recovering"
}
case eval.Alerting:
if alertingRule.ActiveAt == nil || alertingRule.ActiveAt.After(activeAt) {
alertingRule.ActiveAt = &activeAt
pkg/services/ngalert/api/tooling/definitions/prom.go
+3 -2
View File
@@ -152,8 +152,9 @@ type AlertingRule struct {
// required: true
Name string `json:"name,omitempty"`
// required: true
Query string `json:"query,omitempty"`
Duration float64 `json:"duration,omitempty"`
Query string `json:"query,omitempty"`
Duration float64 `json:"duration,omitempty"`
KeepFiringFor float64 `json:"keepFiringFor,omitempty"`
// required: true
Annotations promlabels.Labels `json:"annotations,omitempty"`
// required: true
pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go
+15 -10
View File
@@ -158,6 +158,9 @@ type ProvisionedAlertRule struct {
// required: true
// swagger:strfmt duration
For model.Duration `json:"for"`
// required: false
// swagger:strfmt duration
KeepFiringFor model.Duration `json:"keep_firing_for"`
// example: {"runbook_url": "https://supercoolrunbook.com/page/13"}
Annotations map[string]string `json:"annotations,omitempty"`
// example: {"team": "sre-team-1"}
@@ -259,19 +262,21 @@ type AlertRuleGroupExport struct {
// AlertRuleExport is the provisioned file export of models.AlertRule.
type AlertRuleExport struct {
UID string `json:"uid,omitempty" yaml:"uid,omitempty"`
Title string `json:"title" yaml:"title" hcl:"name"`
Condition *string `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
Data []AlertQueryExport `json:"data" yaml:"data" hcl:"data,block"`
DashboardUID *string `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
PanelID *int64 `json:"panelId,omitempty" yaml:"panelId,omitempty"`
NoDataState *NoDataState `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
ExecErrState *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
For model.Duration `json:"for,omitempty" yaml:"for,omitempty"`
// ForString is used to:
UID string `json:"uid,omitempty" yaml:"uid,omitempty"`
Title string `json:"title" yaml:"title" hcl:"name"`
Condition *string `json:"condition,omitempty" yaml:"condition,omitempty" hcl:"condition"`
Data []AlertQueryExport `json:"data" yaml:"data" hcl:"data,block"`
DashboardUID *string `json:"dashboardUid,omitempty" yaml:"dashboardUid,omitempty"`
PanelID *int64 `json:"panelId,omitempty" yaml:"panelId,omitempty"`
NoDataState *NoDataState `json:"noDataState,omitempty" yaml:"noDataState,omitempty" hcl:"no_data_state"`
ExecErrState *ExecutionErrorState `json:"execErrState,omitempty" yaml:"execErrState,omitempty" hcl:"exec_err_state"`
For model.Duration `json:"for,omitempty" yaml:"for,omitempty"`
KeepFiringFor model.Duration `json:"keepFiringFor,omitempty" yaml:"keepFiringFor,omitempty" hcl:"keep_firing_for"`
// ForString and KeepFiringForString are used to:
// - Only export the for field for HCL if it is non-zero.
// - Format the Prometheus model.Duration type properly for HCL.
ForString *string `json:"-" yaml:"-" hcl:"for"`
KeepFiringForString *string `json:"-" yaml:"-" hcl:"keep_firing_for"`
Annotations *map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty" hcl:"annotations"`
Labels *map[string]string `json:"labels,omitempty" yaml:"labels,omitempty" hcl:"labels"`
IsPaused bool `json:"isPaused" yaml:"isPaused" hcl:"is_paused"`
pkg/services/ngalert/api/validation/api_ruler_validation.go
+21
View File
@@ -157,6 +157,11 @@ func validateAlertingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
return ngmodels.AlertRule{}, err
}
newRule.KeepFiringFor, err = validateKeepFiringForInterval(in)
if err != nil {
return ngmodels.AlertRule{}, err
}
return newRule, nil
}
@@ -185,6 +190,7 @@ func validateRecordingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
newRule.ExecErrState = ""
newRule.Condition = ""
newRule.For = 0
newRule.KeepFiringFor = 0
newRule.NotificationSettings = nil
return newRule, nil
@@ -272,6 +278,21 @@ func validateForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Dur
return duration, nil
}
// validateKeepFiringForInterval validates ApiRuleNode.KeepFiringFor and converts it to time.Duration. If the field is not specified returns 0 if GrafanaManagedAlert.UID is empty and -1 if it is not.
func validateKeepFiringForInterval(ruleNode *apimodels.PostableExtendedRuleNode) (time.Duration, error) {
if ruleNode.ApiRuleNode == nil || ruleNode.ApiRuleNode.KeepFiringFor == nil {
if ruleNode.GrafanaManagedAlert.UID != "" {
return -1, nil // will be patched later with the real value of the current version of the rule
}
return 0, nil // if it's a new rule, use the 0 as the default
}
duration := time.Duration(*ruleNode.ApiRuleNode.KeepFiringFor)
if duration < 0 {
return 0, fmt.Errorf("field `keep_firing_for` cannot be negative [%v]. only 0 or any positive value is allowed", *ruleNode.ApiRuleNode.KeepFiringFor)
}
return duration, nil
}
// ValidateRuleGroup validates API model (definitions.PostableRuleGroupConfig) and converts it to a collection of models.AlertRule.
// Returns a slice that contains all rules described by API model or error if either group specification or an alert definition is not valid.
// It also returns a map containing current existing alerts that don't contain the is_paused field in the body of the call.
pkg/services/ngalert/eval/eval.go
+8 -1
View File
@@ -295,6 +295,11 @@ const (
// Error is the eval state for an alert rule condition
// that evaluated to Error.
Error
// Recovering is the eval state for an alert instance condition
// that evaluated to false (Normal) but has not yet met
// the KeepFiringFor duration defined in AlertRule.
Recovering
)
func (s State) IsValid() bool {
@@ -302,7 +307,7 @@ func (s State) IsValid() bool {
}
func (s State) String() string {
return [...]string{"Normal", "Alerting", "Pending", "NoData", "Error"}[s]
return [...]string{"Normal", "Alerting", "Pending", "NoData", "Error", "Recovering"}[s]
}
func ParseStateString(repr string) (State, error) {
@@ -317,6 +322,8 @@ func ParseStateString(repr string) (State, error) {
return NoData, nil
case "error":
return Error, nil
case "recovering":
return Recovering, nil
default:
return -1, fmt.Errorf("invalid state: %s", repr)
}
pkg/services/ngalert/models/alert_rule.go
+10
View File
@@ -289,6 +289,7 @@ type AlertRule struct {
// ideally this field should have been apimodels.ApiDuration
// but this is currently not possible because of circular dependencies
For time.Duration
KeepFiringFor time.Duration
Annotations map[string]string
Labels map[string]string
IsPaused bool
@@ -648,6 +649,10 @@ func (alertRule *AlertRule) ValidateAlertRule(cfg setting.UnifiedAlertingSetting
return fmt.Errorf("%w: field `for` cannot be negative", ErrAlertRuleFailedValidation)
}
if alertRule.KeepFiringFor < 0 {
return fmt.Errorf("%w: field `keep_firing_for` cannot be negative", ErrAlertRuleFailedValidation)
}
if len(alertRule.Labels) > 0 {
for label := range alertRule.Labels {
if _, ok := LabelsUserCannotSpecify[label]; ok {
@@ -748,6 +753,7 @@ func (alertRule *AlertRule) Copy() *AlertRule {
Record: alertRule.Record,
IsPaused: alertRule.IsPaused,
Metadata: alertRule.Metadata,
KeepFiringFor: alertRule.KeepFiringFor,
MissingSeriesEvalsToResolve: alertRule.MissingSeriesEvalsToResolve,
}
@@ -810,6 +816,7 @@ func ClearRecordingRuleIgnoredFields(rule *AlertRule) {
rule.ExecErrState = ""
rule.Condition = ""
rule.For = 0
rule.KeepFiringFor = 0
rule.NotificationSettings = nil
rule.MissingSeriesEvalsToResolve = nil
}
@@ -959,6 +966,9 @@ func PatchPartialAlertRule(existingRule *AlertRule, ruleToPatch *AlertRuleWithOp
if ruleToPatch.For == -1 {
ruleToPatch.For = existingRule.For
}
if ruleToPatch.KeepFiringFor == -1 {
ruleToPatch.KeepFiringFor = existingRule.KeepFiringFor
}
if !ruleToPatch.HasPause {
ruleToPatch.IsPaused = existingRule.IsPaused
}
pkg/services/ngalert/models/alert_rule_test.go
+49
View File
@@ -530,6 +530,13 @@ func TestDiff(t *testing.T) {
assert.Equal(t, rule2.For, diff[0].Right.Interface())
difCnt++
}
if rule1.KeepFiringFor != rule2.KeepFiringFor {
diff := diffs.GetDiffsForField("KeepFiringFor")
assert.Len(t, diff, 1)
assert.Equal(t, rule1.KeepFiringFor, diff[0].Left.Interface())
assert.Equal(t, rule2.KeepFiringFor, diff[0].Right.Interface())
difCnt++
}
if rule1.RuleGroupIndex != rule2.RuleGroupIndex {
diff := diffs.GetDiffsForField("RuleGroupIndex")
assert.Len(t, diff, 1)
@@ -1044,6 +1051,48 @@ func TestGeneratorFillsAllFields(t *testing.T) {
require.FailNow(t, "AlertRule generator does not populate fields", "skipped fields: %v", maps.Keys(fields))
}
func TestValidateAlertRule(t *testing.T) {
testCases := []struct {
name string
keepFiringFor time.Duration
expectedErr error
}{
{
name: "should accept zero keep firing for",
keepFiringFor: 0,
expectedErr: nil,
},
{
name: "should accept positive keep firing for",
keepFiringFor: 1 * time.Minute,
expectedErr: nil,
},
{
name: "should reject negative keep firing for",
keepFiringFor: -1 * time.Minute,
expectedErr: fmt.Errorf("%w: field `keep_firing_for` cannot be negative", ErrAlertRuleFailedValidation),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
rule := RuleGen.With(
RuleGen.WithKeepFiringFor(tc.keepFiringFor),
RuleGen.WithIntervalSeconds(10),
).GenerateRef()
err := rule.ValidateAlertRule(setting.UnifiedAlertingSettings{BaseInterval: 10 * time.Second})
if tc.expectedErr == nil {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Equal(t, tc.expectedErr.Error(), err.Error())
}
})
}
}
func TestAlertRule_PrometheusRuleDefinition(t *testing.T) {
tests := []struct {
name string
pkg/services/ngalert/models/instance.go
+4 -1
View File
@@ -39,6 +39,8 @@ const (
InstanceStateNoData InstanceStateType = "NoData"
// InstanceStateError is for an erroring alert.
InstanceStateError InstanceStateType = "Error"
// InstanceStateRecovering is for a recovering alert.
InstanceStateRecovering InstanceStateType = "Recovering"
)
// IsValid checks that the value of InstanceStateType is a valid
@@ -48,7 +50,8 @@ func (i InstanceStateType) IsValid() bool {
i == InstanceStateNormal ||
i == InstanceStateNoData ||
i == InstanceStatePending ||
i == InstanceStateError
i == InstanceStateError ||
i == InstanceStateRecovering
}
// ListAlertInstancesQuery is the query list alert Instances.
pkg/services/ngalert/models/instance_test.go
+4
View File
@@ -33,6 +33,10 @@ func TestInstanceStateType_IsValid(t *testing.T) {
instanceType: InstanceStateError,
expectedValidity: true,
},
{
instanceType: InstanceStateRecovering,
expectedValidity: true,
},
{
instanceType: InstanceStateType("notAValidInstanceStateType"),
expectedValidity: false,
pkg/services/ngalert/models/testing.go
+15
View File
@@ -73,6 +73,7 @@ func (g *AlertRuleGenerator) Generate() AlertRule {
interval := (rand.Int63n(6) + 1) * 10
forInterval := time.Duration(interval*rand.Int63n(6)) * time.Second
keepFiringFor := time.Duration(interval*rand.Int63n(6)) * time.Second
var annotations map[string]string = nil
if rand.Int63()%2 == 0 {
@@ -122,6 +123,7 @@ func (g *AlertRuleGenerator) Generate() AlertRule {
NoDataState: randNoDataState(),
ExecErrState: randErrState(),
For: forInterval,
KeepFiringFor: keepFiringFor,
Annotations: annotations,
Labels: labels,
NotificationSettings: ns,
@@ -344,6 +346,18 @@ func (a *AlertRuleMutators) WithForNTimes(timesOfInterval int64) AlertRuleMutato
}
}
func (a *AlertRuleMutators) WithKeepFiringFor(interval time.Duration) AlertRuleMutator {
return func(rule *AlertRule) {
rule.KeepFiringFor = interval
}
}
func (a *AlertRuleMutators) WithKeepFiringForNTimes(timesOfInterval int64) AlertRuleMutator {
return func(rule *AlertRule) {
rule.KeepFiringFor = time.Duration(rule.IntervalSeconds*timesOfInterval) * time.Second
}
}
func (a *AlertRuleMutators) WithNoDataExecAs(nodata NoDataState) AlertRuleMutator {
return func(rule *AlertRule) {
rule.NoDataState = nodata
@@ -855,6 +869,7 @@ func AlertInstanceGen(mutators ...AlertInstanceMutator) *AlertInstance {
InstanceStatePending,
InstanceStateNoData,
InstanceStateError,
InstanceStateRecovering,
}
return s[rand.Intn(len(s))]
}
pkg/services/ngalert/schedule/alert_rule_test.go
+5 -1
View File
@@ -847,7 +847,11 @@ func TestRuleRoutine(t *testing.T) {
})
t.Run("when there are resolved alerts they should keep sending until retention period is over", func(t *testing.T) {
rule := gen.With(withQueryForState(t, eval.Normal), models.RuleMuts.WithInterval(time.Second)).GenerateRef()
rule := gen.With(
withQueryForState(t, eval.Normal),
models.RuleMuts.WithInterval(time.Second),
models.RuleMuts.WithKeepFiringFor(0),
).GenerateRef()
evalAppliedChan := make(chan time.Time)
pkg/services/ngalert/schedule/registry_test.go
+2
View File
@@ -196,6 +196,7 @@ func TestRuleWithFolderFingerprint(t *testing.T) {
ExecErrState: "test-err",
Record: &models.Record{Metric: "my_metric", From: "A"},
For: 12,
KeepFiringFor: 456,
Annotations: map[string]string{
"key-annotation": "value-annotation",
},
@@ -242,6 +243,7 @@ func TestRuleWithFolderFingerprint(t *testing.T) {
ExecErrState: "test-err2",
Record: &models.Record{Metric: "my_metric2", From: "B"},
For: 1141,
KeepFiringFor: 123,
Annotations: map[string]string{
"key-annotation2": "value-annotation",
},
pkg/services/ngalert/state/manager.go
+4 -1
View File
@@ -249,7 +249,7 @@ func (st *Manager) DeleteStateByRuleUID(ctx context.Context, ruleKey ngModels.Al
s.SetNormal(reason, startsAt, now)
// Set Resolved property so the scheduler knows to send a postable alert
// to Alertmanager.
if oldState == eval.Alerting || oldState == eval.Error || oldState == eval.NoData {
if oldState == eval.Alerting || oldState == eval.Error || oldState == eval.NoData || oldState == eval.Recovering {
s.ResolvedAt = &now
} else {
s.ResolvedAt = nil
@@ -510,6 +510,8 @@ func translateInstanceState(state ngModels.InstanceStateType) eval.State {
return eval.NoData
case ngModels.InstanceStatePending:
return eval.Pending
case ngModels.InstanceStateRecovering:
return eval.Recovering
default:
return eval.Error
}
@@ -582,6 +584,7 @@ func StatesToRuleStatus(states []*State) ngModels.RuleStatus {
case eval.Normal:
case eval.Pending:
case eval.Alerting:
case eval.Recovering:
case eval.Error:
status.Health = "error"
case eval.NoData:
pkg/services/ngalert/state/manager_private_test.go
+568
View File
@@ -778,6 +778,142 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
{
desc: "t1[1:alerting] t2[1:normal] t3[1:normal] t4[1:normal] and 'keep_firing_for'>0 at t2,t3,t4",
alertRule: baseRuleWith(ngmodels.RuleMuts.WithKeepFiringForNTimes(2), ngmodels.RuleMuts.WithFor(0)),
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t4: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
},
expectedTransitions: map[time.Time][]StateTransition{
t1: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
LastSentAt: &t1,
},
},
},
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Recovering,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Recovering,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Recovering,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Recovering,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t4,
EndsAt: t4,
LastEvaluationTime: t4,
LastSentAt: &t4,
ResolvedAt: &t4,
},
},
},
},
},
{
desc: "t1[1alerting] t2[1:normal] t3[1:alerting] and 'keep_firing_for'>0 at t2,t3",
alertRule: baseRuleWith(ngmodels.RuleMuts.WithKeepFiringForNTimes(1), ngmodels.RuleMuts.WithFor(0)),
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
},
expectedTransitions: map[time.Time][]StateTransition{
t1: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
LastSentAt: &t1,
},
},
},
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Recovering,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Recovering,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
},
},
{
desc: "t1[1:alerting] t2[NoData] t3[NoData] at t2,t3",
alertRule: baseRule,
@@ -1916,6 +2052,244 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
{
desc: "t1[{}:alerting] t2[NoData] t3[NoData] t4[NoData] and 'keep_firing_for'=2 at t2,t3,t4",
ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.RuleMuts.WithKeepFiringForNTimes(2)},
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
},
t3: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
},
t4: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
},
},
expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.NoData: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
Annotations: baseRule.Annotations,
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
},
},
},
t3: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
Annotations: baseRule.Annotations,
State: eval.NoData,
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t2,
},
},
},
t4: {
{
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
Annotations: baseRule.Annotations,
State: eval.NoData,
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t2,
EndsAt: t4.Add(ResendDelay * 4),
LastEvaluationTime: t4,
LastSentAt: &t2,
},
},
},
},
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t1,
EndsAt: t4.Add(ResendDelay * 4),
LastEvaluationTime: t4,
LastSentAt: &t4,
},
},
},
},
ngmodels.OK: {
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Recovering,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Recovering,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Recovering,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Recovering,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t4,
EndsAt: t4,
LastEvaluationTime: t4,
LastSentAt: &t4,
ResolvedAt: &t4,
},
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Alerting,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Alerting,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t1,
EndsAt: t4.Add(ResendDelay * 4),
LastEvaluationTime: t4,
LastSentAt: &t4,
},
},
},
},
},
},
{
desc: "t1[NoData] t2[1:normal] t3[1:normal] at t3",
results: map[time.Time]eval.Results{
@@ -2964,6 +3338,200 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
{
desc: "t1[{}:QueryError] t2[{}:normal] t3[{}:normal] t4[{}:normal] and 'keep_firing_for'=2 at t2,t3,t4",
ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.RuleMuts.WithKeepFiringForNTimes(2)},
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithError(datasourceError)),
},
t2: {
newResult(eval.WithState(eval.Normal)),
},
t3: {
newResult(eval.WithState(eval.Normal)),
},
t4: {
newResult(eval.WithState(eval.Normal)),
},
},
expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
ngmodels.ErrorErrState: {
t2: {
{
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t4,
LastSentAt: &t1,
},
},
},
},
ngmodels.AlertingErrState: {
t2: {
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.Error.String(),
State: &State{
Labels: labels["system + rule"],
State: eval.Recovering,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Recovering,
State: &State{
Labels: labels["system + rule"],
State: eval.Recovering,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t1,
},
},
},
t4: {
{
PreviousState: eval.Recovering,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t4,
EndsAt: t4,
LastEvaluationTime: t4,
ResolvedAt: &t4,
LastSentAt: &t4,
},
},
},
},
ngmodels.OkErrState: {
t2: {
{
PreviousState: eval.Normal,
PreviousStateReason: eval.Error.String(),
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
},
},
t4: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t4,
},
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
},
},
t4: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t4,
},
},
},
},
},
},
{
desc: "t1[{}:normal] t2[QueryError] at t2",
results: map[time.Time]eval.Results{
pkg/services/ngalert/state/manager_test.go
+119
View File
@@ -222,6 +222,25 @@ func TestWarmStateCache(t *testing.T) {
Labels: labels,
ResultFingerprint: data.Fingerprint(2).String(),
})
labels = models.InstanceLabels{"test6": "testValue6"}
_, hash, _ = labels.StringAndHash()
instances = append(instances, models.AlertInstance{
AlertInstanceKey: models.AlertInstanceKey{
RuleOrgID: rule.OrgID,
RuleUID: rule.UID,
LabelsHash: hash,
},
CurrentState: models.InstanceStateRecovering,
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: nil,
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(2).String(),
})
for _, instance := range instances {
_ = ng.InstanceStore.SaveAlertInstance(ctx, instance)
}
@@ -1082,6 +1101,106 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> alerting -> normal (recovering) when KeepFiringFor is set but is not exceeded",
alertRule: baseRuleWith(m.WithKeepFiringForNTimes(2)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Recovering,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: util.Pointer(t2),
},
},
},
{
desc: "normal -> alerting -> normal (recovering) -> normal (recovering) -> normal (resolved) when KeepFiringFor is set and has passed",
alertRule: baseRuleWith(m.WithKeepFiringForNTimes(2)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
tn(5): {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(tn(5), eval.Normal),
StartsAt: tn(5),
EndsAt: tn(5),
LastEvaluationTime: tn(5),
LastSentAt: util.Pointer(tn(5)),
ResolvedAt: util.Pointer(tn(5)),
},
},
},
{
desc: "normal -> alerting -> normal(recovering) -> alerting -> alerting -> recovering when KeepFiringFor is set",
alertRule: baseRuleWith(m.WithKeepFiringForNTimes(3)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(5): {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(6): {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 4,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Recovering,
LatestResult: newEvaluation(tn(6), eval.Normal),
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
LastSentAt: util.Pointer(tn(5)),
},
},
},
}
for _, tc := range testCases {
pkg/services/ngalert/state/state.go
+60 -9
View File
@@ -68,7 +68,7 @@ type State struct {
// and states that have been resolved. It cannot be used to determine when a state was resolved.
EndsAt time.Time
// ResolvedAt is set when the state is first resolved. That is to say, when the state first transitions
// from Alerting, NoData, or Error to Normal. It is reset to zero when the state transitions from Normal
// from Alerting, NoData, Recovering, or Error to Normal. It is reset to zero when the state transitions from Normal
// to any other state.
ResolvedAt *time.Time
LastSentAt *time.Time
@@ -161,7 +161,7 @@ func (a *State) SetAlerting(reason string, startsAt, endsAt time.Time) {
a.Error = nil
}
// SetPending the state to Pending. It changes both the start and end time.
// SetPending sets the state to Pending. It changes both the start and end time.
func (a *State) SetPending(reason string, startsAt, endsAt time.Time) {
a.State = eval.Pending
a.StateReason = reason
@@ -170,6 +170,15 @@ func (a *State) SetPending(reason string, startsAt, endsAt time.Time) {
a.Error = nil
}
// SetRecovering sets the state to Recovering. It changes both the start and end time.
func (a *State) SetRecovering(reason string, startsAt, endsAt time.Time) {
a.State = eval.Recovering
a.StateReason = reason
a.StartsAt = startsAt
a.EndsAt = endsAt
a.Error = nil
}
// SetNoData sets the state to NoData. It changes both the start and end time.
func (a *State) SetNoData(reason string, startsAt, endsAt time.Time) {
a.State = eval.NoData
@@ -319,10 +328,50 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]float6
return result
}
func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger log.Logger, reason string) {
if state.State == eval.Normal {
func resultNormal(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger, reason string) {
switch {
case state.State == eval.Normal:
logger.Debug("Keeping state", "state", state.State)
} else {
case state.State == eval.Recovering:
// If the previous state is Recovering then check if the KeepFiringFor duration has been observed,
// and if so, transition to Normal.
if result.EvaluatedAt.Sub(state.StartsAt) >= rule.KeepFiringFor {
nextEndsAt := result.EvaluatedAt
logger.Debug("Changing state",
"previous_state",
state.State,
"next_state",
eval.Normal,
"previous_ends_at",
state.EndsAt,
"next_ends_at",
nextEndsAt,
)
state.SetNormal(reason, nextEndsAt, nextEndsAt)
} else {
// If the KeepFiringFor duration has not been observed then the state is kept as Recovering.
// We must also set the next endsAt to a future time for the Alertmanager,
// as for it the alert is still firing.
state.EndsAt = nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)
}
case state.State == eval.Alerting && rule.KeepFiringFor > 0:
// If the old state is Alerting and the rule has a KeepFiringFor duration then
// the state should be set to Recovering when it transitions to Normal.
//
// EndsAt must be set to a future time for the Alertmanager, the same as for Alerting states.
nextEndsAt := nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)
logger.Debug("Changing state",
"previous_state",
state.State,
"next_state",
eval.Recovering,
"previous_ends_at",
state.EndsAt,
"next_ends_at",
nextEndsAt,
)
state.SetRecovering(reason, result.EvaluatedAt, nextEndsAt)
default:
nextEndsAt := result.EvaluatedAt
logger.Debug("Changing state",
"previous_state",
@@ -332,7 +381,8 @@ func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger
"previous_ends_at",
state.EndsAt,
"next_ends_at",
nextEndsAt)
nextEndsAt,
)
// Normal states have the same start and end timestamps
state.SetNormal(reason, nextEndsAt, nextEndsAt)
}
@@ -789,8 +839,9 @@ func (a *State) transition(alertRule *models.AlertRule, result eval.Result, extr
case eval.NoData:
logger.Debug("Setting next state", "handler", "resultNoData")
resultNoData(a, alertRule, result, logger)
case eval.Pending: // we do not emit results with this state
logger.Debug("Ignoring set next state as result is pending")
case eval.Pending,
eval.Recovering: // we do not emit results with these states
logger.Debug("Ignoring set next state", "state", result.State)
}
// Set reason iff: result and state are different, reason is not Alerting or Normal
@@ -805,7 +856,7 @@ func (a *State) transition(alertRule *models.AlertRule, result eval.Result, extr
// Set Resolved property so the scheduler knows to send a postable alert
// to Alertmanager.
newlyResolved := false
if oldState == eval.Alerting && a.State == eval.Normal {
if oldState == eval.Alerting && a.State == eval.Normal || oldState == eval.Recovering && a.State == eval.Normal {
a.ResolvedAt = &result.EvaluatedAt
newlyResolved = true
} else if a.State != eval.Normal && a.State != eval.Pending { // Retain the last resolved time for Normal->Normal and Normal->Pending.
pkg/services/ngalert/store/compat.go
+4
View File
@@ -34,6 +34,7 @@ func alertRuleToModelsAlertRule(ar alertRule, l log.Logger) (models.AlertRule, e
RuleGroup: ar.RuleGroup,
RuleGroupIndex: ar.RuleGroupIndex,
For: ar.For,
KeepFiringFor: ar.KeepFiringFor,
IsPaused: ar.IsPaused,
MissingSeriesEvalsToResolve: ar.MissingSeriesEvalsToResolve,
}
@@ -125,6 +126,7 @@ func alertRuleFromModelsAlertRule(ar models.AlertRule) (alertRule, error) {
NoDataState: ar.NoDataState.String(),
ExecErrState: ar.ExecErrState.String(),
For: ar.For,
KeepFiringFor: ar.KeepFiringFor,
IsPaused: ar.IsPaused,
MissingSeriesEvalsToResolve: ar.MissingSeriesEvalsToResolve,
}
@@ -202,6 +204,7 @@ func alertRuleToAlertRuleVersion(rule alertRule) alertRuleVersion {
NoDataState: rule.NoDataState,
ExecErrState: rule.ExecErrState,
For: rule.For,
KeepFiringFor: rule.KeepFiringFor,
Annotations: rule.Annotations,
Labels: rule.Labels,
IsPaused: rule.IsPaused,
@@ -235,6 +238,7 @@ func alertRuleVersionToAlertRule(version alertRuleVersion) alertRule {
NoDataState: version.NoDataState,
ExecErrState: version.ExecErrState,
For: version.For,
KeepFiringFor: version.KeepFiringFor,
Annotations: version.Annotations,
Labels: version.Labels,
IsPaused: version.IsPaused,
pkg/services/ngalert/store/models.go
+2
View File
@@ -24,6 +24,7 @@ type alertRule struct {
NoDataState string
ExecErrState string
For time.Duration
KeepFiringFor time.Duration
Annotations string
Labels string
IsPaused bool
@@ -61,6 +62,7 @@ type alertRuleVersion struct {
// ideally this field should have been apimodels.ApiDuration
// but this is currently not possible because of circular dependencies
For time.Duration
KeepFiringFor time.Duration
Annotations string
Labels string
IsPaused bool
pkg/services/sqlstore/migrations/migrations.go
+2
View File
@@ -148,6 +148,8 @@ func (oss *OSSMigrations) AddMigration(mg *Migrator) {
ualert.AddAlertRuleGuidMigration(mg)
ualert.AddAlertRuleKeepFiringFor(mg)
ualert.AddAlertRuleMissingSeriesEvalsToResolve(mg)
ualert.AddAlertRuleVersionUIDIndex(mg)
pkg/services/sqlstore/migrations/ualert/alert_rule_keep_firing_for.go
+17
View File
@@ -0,0 +1,17 @@
package ualert
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
// AddAlertRuleKeepFiringFor adds keep_firing_for column to alert_rule and alert_rule_version tables.
func AddAlertRuleKeepFiringFor(mg *migrator.Migrator) {
column := &migrator.Column{Name: "keep_firing_for", Type: migrator.DB_BigInt, Nullable: false, Default: "0"}
mg.AddMigration(
"add keep_firing_for column to alert_rule",
migrator.NewAddColumnMigration(migrator.Table{Name: "alert_rule"}, column),
)
mg.AddMigration(
"add keep_firing_for column to alert_rule_version",
migrator.NewAddColumnMigration(migrator.Table{Name: "alert_rule_version"}, column),
)
}
pkg/tests/api/alerting/api_prometheus_test.go
+8 -2
View File
@@ -379,6 +379,9 @@ func TestIntegrationPrometheusRulesFilterByDashboard(t *testing.T) {
interval, err := model.ParseDuration("10s")
require.NoError(t, err)
keepFiringFor, err := model.ParseDuration("15s")
require.NoError(t, err)
// Now, let's create some rules
{
rules := apimodels.PostableRuleGroupConfig{
@@ -386,8 +389,9 @@ func TestIntegrationPrometheusRulesFilterByDashboard(t *testing.T) {
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &interval,
Labels: map[string]string{},
For: &interval,
KeepFiringFor: &keepFiringFor,
Labels: map[string]string{},
Annotations: map[string]string{
"__dashboardUid__": dashboardUID,
"__panelId__": "1",
@@ -472,6 +476,7 @@ func TestIntegrationPrometheusRulesFilterByDashboard(t *testing.T) {
"folderUid": "default",
"query": "[{\"refId\":\"A\",\"queryType\":\"\",\"relativeTimeRange\":{\"from\":18000,\"to\":10800},\"datasourceUid\":\"__expr__\",\"model\":{\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":43200,\"type\":\"math\"}}]",
"duration": 10,
"keepFiringFor": 15,
"annotations": {
"__dashboardUid__": "%s",
"__panelId__": "1"
@@ -518,6 +523,7 @@ func TestIntegrationPrometheusRulesFilterByDashboard(t *testing.T) {
"folderUid": "default",
"query": "[{\"refId\":\"A\",\"queryType\":\"\",\"relativeTimeRange\":{\"from\":18000,\"to\":10800},\"datasourceUid\":\"__expr__\",\"model\":{\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":43200,\"type\":\"math\"}}]",
"duration": 10,
"keepFiringFor": 15,
"annotations": {
"__dashboardUid__": "%s",
"__panelId__": "1"
pkg/tests/api/alerting/api_ruler_test.go
+59 -10
View File
@@ -813,9 +813,10 @@ func TestIntegrationAlertRuleEditorSettings(t *testing.T) {
require.NoError(t, err)
alertRule := apimodels.PostableExtendedRuleNode{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &interval,
Labels: map[string]string{"label1": "val1"},
Annotations: map[string]string{"annotation1": "val1"},
For: &interval,
KeepFiringFor: &interval,
Labels: map[string]string{"label1": "val1"},
Annotations: map[string]string{"annotation1": "val1"},
},
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
Title: "AlwaysFiring",
@@ -1088,8 +1089,9 @@ func TestIntegrationRulerRulesFilterByDashboard(t *testing.T) {
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &interval,
Labels: map[string]string{},
For: &interval,
KeepFiringFor: &interval,
Labels: map[string]string{},
Annotations: map[string]string{
"__dashboardUid__": dashboardUID,
"__panelId__": "1",
@@ -1151,6 +1153,7 @@ func TestIntegrationRulerRulesFilterByDashboard(t *testing.T) {
"rules": [{
"expr": "",
"for": "10s",
"keep_firing_for": "10s",
"annotations": {
"__dashboardUid__": "%s",
"__panelId__": "1"
@@ -1197,6 +1200,7 @@ func TestIntegrationRulerRulesFilterByDashboard(t *testing.T) {
}, {
"expr": "",
"for":"0s",
"keep_firing_for": "0s",
"grafana_alert": {
"title": "AlwaysFiringButSilenced",
"condition": "A",
@@ -1247,6 +1251,7 @@ func TestIntegrationRulerRulesFilterByDashboard(t *testing.T) {
"rules": [{
"expr": "",
"for": "10s",
"keep_firing_for": "10s",
"annotations": {
"__dashboardUid__": "%s",
"__panelId__": "1"
@@ -1556,7 +1561,8 @@ func TestIntegrationRuleCreate(t *testing.T) {
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: util.Pointer(model.Duration(2 * time.Minute)),
For: util.Pointer(model.Duration(2 * time.Minute)),
KeepFiringFor: util.Pointer(model.Duration(1 * time.Minute)),
Labels: map[string]string{
"foo🙂": "bar",
"_bar1": "baz🙂",
@@ -1589,7 +1595,8 @@ func TestIntegrationRuleCreate(t *testing.T) {
Rules: []apimodels.GettableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: util.Pointer(model.Duration(2 * time.Minute)),
For: util.Pointer(model.Duration(2 * time.Minute)),
KeepFiringFor: util.Pointer(model.Duration(1 * time.Minute)),
Labels: map[string]string{
"foo🙂": "bar",
"_bar1": "baz🙂",
@@ -1733,6 +1740,27 @@ func TestIntegrationRuleUpdate(t *testing.T) {
require.Equal(t, http.StatusAccepted, status)
require.Equal(t, expected, *getGroup.Rules[0].ApiRuleNode.For)
})
t.Run("should be able to reset 'keep_firing_for' to 0", func(t *testing.T) {
group := generateAlertRuleGroup(1, alertRuleGen())
keepFiringFor := model.Duration(10 * time.Second)
group.Rules[0].ApiRuleNode.KeepFiringFor = &keepFiringFor
_, status, body := client.PostRulesGroupWithStatus(t, folderUID, &group, false)
require.Equalf(t, http.StatusAccepted, status, "failed to post rule group. Response: %s", body)
getGroup, _ := client.GetRulesGroup(t, folderUID, group.Name)
require.Equal(t, keepFiringFor, *getGroup.Rules[0].ApiRuleNode.KeepFiringFor)
group = convertGettableRuleGroupToPostable(getGroup.GettableRuleGroupConfig)
newKeepFiringFor := model.Duration(0)
group.Rules[0].ApiRuleNode.KeepFiringFor = &newKeepFiringFor
_, status, body = client.PostRulesGroupWithStatus(t, folderUID, &group, false)
require.Equalf(t, http.StatusAccepted, status, "failed to post rule group. Response: %s", body)
getGroup, _ = client.GetRulesGroup(t, folderUID, group.Name)
require.Equal(t, newKeepFiringFor, *getGroup.Rules[0].ApiRuleNode.KeepFiringFor)
})
t.Run("when data source missing", func(t *testing.T) {
var groupName string
{
@@ -2552,6 +2580,7 @@ func TestIntegrationQuota(t *testing.T) {
{
"expr":"",
"for": "2m",
"keep_firing_for": "0s",
"grafana_alert":{
"title":"Updated alert rule",
"condition":"A",
@@ -2660,6 +2689,7 @@ func TestIntegrationDeleteFolderWithRules(t *testing.T) {
{
"expr": "",
"for": "2m",
"keep_firing_for": "0s",
"labels": {
"label1": "val1"
},
@@ -3146,6 +3176,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
},
"expr":"",
"for": "1m",
"keep_firing_for": "0s",
"labels": {
"label1": "val1"
},
@@ -3194,6 +3225,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
{
"expr":"",
"for": "0s",
"keep_firing_for": "0s",
"grafana_alert":{
"title":"AlwaysFiringButSilenced",
"condition":"A",
@@ -3250,6 +3282,9 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
interval, err := model.ParseDuration("30s")
require.NoError(t, err)
keepFiringFor, err := model.ParseDuration("10s")
require.NoError(t, err)
rules := apimodels.PostableRuleGroupConfig{
Name: "arulegroup",
Rules: []apimodels.PostableExtendedRuleNode{
@@ -3306,7 +3341,8 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
},
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &interval,
For: &interval,
KeepFiringFor: &keepFiringFor,
Labels: map[string]string{
"label1": "val42",
"foo": "bar",
@@ -3537,6 +3573,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
},
"expr":"",
"for": "1m",
"keep_firing_for": "0s",
"labels": {
"label1": "val1"
},
@@ -3585,6 +3622,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
{
"expr":"",
"for": "0s",
"keep_firing_for": "0s",
"grafana_alert":{
"title":"AlwaysFiringButSilenced",
"condition":"A",
@@ -3639,12 +3677,16 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
forValue, err := model.ParseDuration("30s")
require.NoError(t, err)
keepFiringForValue, err := model.ParseDuration("5s")
require.NoError(t, err)
rules := apimodels.PostableRuleGroupConfig{
Name: "arulegroup",
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &forValue,
For: &forValue,
KeepFiringFor: &keepFiringForValue,
Labels: map[string]string{
// delete foo label
"label1": "val1", // update label value
@@ -3719,6 +3761,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
},
"expr":"",
"for": "30s",
"keep_firing_for": "5s",
"labels": {
"label1": "val1",
"label2": "val2"
@@ -3776,12 +3819,16 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
forValue, err := model.ParseDuration("30s")
require.NoError(t, err)
keepFiringForValue, err := model.ParseDuration("15s")
require.NoError(t, err)
rules := apimodels.PostableRuleGroupConfig{
Name: "arulegroup",
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: &forValue,
For: &forValue,
KeepFiringFor: &keepFiringForValue,
},
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
UID: ruleUID, // Including the UID in the payload makes the endpoint update the existing rule.
@@ -3841,6 +3888,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
{
"expr":"",
"for": "30s",
"keep_firing_for": "15s",
"grafana_alert":{
"title":"AlwaysNormal",
"condition":"A",
@@ -3938,6 +3986,7 @@ func TestIntegrationAlertRuleCRUD(t *testing.T) {
{
"expr":"",
"for": "30s",
"keep_firing_for": "15s",
"grafana_alert":{
"title":"AlwaysNormal",
"condition":"A",
pkg/tests/api/alerting/test-data/rulegroup-1-get.json
+2
View File
@@ -5,6 +5,7 @@
{
"expr": "",
"for": "5m",
"keep_firing_for": "0s",
"labels": {
"label1": "test-label"
},
@@ -51,6 +52,7 @@
{
"expr": "",
"for": "5m",
"keep_firing_for": "0s",
"labels": {
"label1": "test-label"
},
pkg/tests/api/alerting/test-data/rulegroup-2-get.json
+1
View File
@@ -5,6 +5,7 @@
{
"expr": "",
"for": "5m",
"keep_firing_for": "0s",
"labels": {
"label1": "test-label"
},
pkg/tests/api/alerting/test-data/rulegroup-3-get.json
+2
View File
@@ -5,6 +5,7 @@
{
"expr": "",
"for": "5m",
"keep_firing_for": "0s",
"labels": {
"label1": "test-label"
},
@@ -51,6 +52,7 @@
{
"expr": "",
"for": "5m",
"keep_firing_for": "0s",
"labels": {
"label1": "test-label"
},