public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Send notifications immediately on Error|NoData -> Normal transitions (#106421)

Browse Source
This commit is contained in:
Alexander Akhmetov
2025-06-10 16:36:30 +02:00
committed by GitHub
parent cbb828202a
commit 1a75787e74
3 changed files with 177 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/state/manager.go
+9 -1
View File
@@ -538,7 +538,15 @@ func (st *Manager) processMissingSeriesStates(logger log.Logger, evaluatedAt tim
// Now we need check if it's stale, and if so, we need to resolve it.
oldState := s.State
oldReason := s.StateReason
isStale := stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds, alertRule.GetMissingSeriesEvalsToResolve())
missingEvalsToResolve := alertRule.GetMissingSeriesEvalsToResolve()
// Error state should be resolved after 1 missing evaluation instead of waiting
// for the configured missing series evaluations. This ensures resolved notifications are sent
// immediately when the alert transitions from these states.
if s.State == eval.Error || s.State == eval.NoData {
missingEvalsToResolve = 1
}
isStale := stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds, missingEvalsToResolve)
if isStale {
logger.Info("Detected stale state entry", "cacheID", s.CacheID, "state", s.State, "reason", s.StateReason)
pkg/services/ngalert/state/manager_private_test.go
+166 -91
View File
@@ -2272,12 +2272,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.NoData,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
EvaluationDuration: time.Millisecond * 10,
},
},
@@ -2668,12 +2670,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
State: &State{
Labels: labels["system + rule + no-data"],
Annotations: baseRule.Annotations,
State: eval.NoData,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t2,
EndsAt: t4,
LastEvaluationTime: t4,
ResolvedAt: &t4,
LastSentAt: &t4,
EvaluationDuration: time.Millisecond * 10,
},
},
@@ -2717,13 +2721,13 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
{
PreviousState: eval.NoData,
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
Annotations: baseRule.Annotations,
State: eval.NoData,
LatestResult: newEvaluation(t5, eval.NoData),
StartsAt: t2,
StartsAt: t5,
EndsAt: t5.Add(ResendDelay * 4),
LastEvaluationTime: t5,
LastSentAt: &t5,
@@ -2881,7 +2885,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
{
desc: "t1[NoData] t2[1:normal] t3[1:normal] at t3",
desc: "t1[NoData] t2[1:normal] t3[1:normal] at t2,t3",
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
@@ -2895,6 +2899,33 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.NoData: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t2,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
@@ -2907,23 +2938,37 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: ngmodels.StateReasonNoData,
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: &t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
},
},
t3: {
{
PreviousState: eval.Normal,
@@ -2955,6 +3000,32 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
ngmodels.OK: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.StateReasonNoData,
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
},
},
t3: {
{
PreviousState: eval.Normal,
@@ -2983,6 +3054,32 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
},
},
t3: {
{
PreviousState: eval.Normal,
@@ -3341,12 +3438,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.NoData,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.NoData,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
EvaluationDuration: time.Millisecond * 10,
},
},
@@ -3844,13 +3943,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Error: datasourceError,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
@@ -4103,24 +4204,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
Error: datasourceError,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
},
},
},
},
ngmodels.AlertingErrState: {
@@ -4240,13 +4323,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Error: datasourceError,
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
LastSentAt: &t1,
EndsAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
ResolvedAt: &t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
@@ -4265,24 +4350,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
t3: {
{
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Normal,
StateReason: "MissingSeries",
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t3,
ResolvedAt: &t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
Error: datasourceError,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
},
},
{
PreviousState: eval.Normal,
State: &State{
@@ -4486,13 +4553,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: "MissingSeries",
Error: datasourceError,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t1,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
@@ -4523,13 +4592,13 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
{
PreviousState: eval.Error,
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
LatestResult: newEvaluation(t4, eval.Error),
Error: datasourceError,
StartsAt: t1,
StartsAt: t4,
EndsAt: t4.Add(ResendDelay * 4),
LastEvaluationTime: t4,
LastSentAt: &t4,
@@ -4953,13 +5022,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Error: datasourceError,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
EndsAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
ResolvedAt: &t3,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
@@ -5097,12 +5168,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
LastSentAt: &t1,
EndsAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
ResolvedAt: &t2,
Error: datasourceError,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -5211,13 +5284,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Error,
State: &State{
Labels: labels["system + rule + datasource-error"],
State: eval.Error,
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Error: datasourceError,
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
LastSentAt: &t1,
EndsAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
ResolvedAt: &t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
}),
pkg/services/ngalert/state/manager_test.go
+2 -28
View File
@@ -885,7 +885,7 @@ func TestProcessEvalResults(t *testing.T) {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 1,
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
@@ -896,16 +896,6 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: t1,
LastEvaluationTime: t3,
},
{
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
},
},
},
{
@@ -1013,10 +1003,9 @@ func TestProcessEvalResults(t *testing.T) {
},
},
{
// TODO(@moustafab): figure out why this test doesn't fail as is
desc: "classic condition, execution Error as Error (alerting -> query error -> alerting)",
alertRule: baseRuleWith(m.WithErrorExecAs(models.ErrorErrState)),
expectedAnnotations: 2,
expectedAnnotations: 3,
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(data.Labels{})),
@@ -1043,21 +1032,6 @@ func TestProcessEvalResults(t *testing.T) {
"annotation": "test",
},
},
{
Labels: data.Labels{"system": "owned", "label": "test", "ref_id": "A", "datasource_uid": "datasource_uid_1"},
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Error,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
Error: expr.MakeQueryError("A", "test-datasource-uid", errors.New("this is an error")),
Annotations: map[string]string{
"Error": "[sse.dataQueryError] failed to execute query [A]: this is an error",
"annotation": "test",
},
},
},
},
{