From c7fdf8ce706c2c9d438f5e6eabd6e580bac4946b Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 15 Aug 2024 12:34:50 +0200 Subject: [PATCH] Alerting: Add error to annotations on data source errors (#91594) --- .../ngalert/state/manager_private_test.go | 88 +++++++++++++++++-- pkg/services/ngalert/state/manager_test.go | 31 +++++-- pkg/services/ngalert/state/state.go | 36 +++++--- 3 files changed, 131 insertions(+), 24 deletions(-) diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 254200f100c..cc592f59400 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -201,6 +201,20 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { "datasource_uid": "1", "ref_id": "A", } + genericErrorAnnotations := mergeLabels( + baseRule.Annotations, + data.Labels{ + "Error": genericError.Error(), + }, + ) + datasourceErrorAnnotations := mergeLabels( + baseRule.Annotations, + data.Labels{ + "datasource_uid": "datasource_uid_1", + "ref_id": "A", + "Error": datasourceError.Error(), + }, + ) labels := map[string]data.Labels{ "system + rule": mergeLabels(baseRule.Labels, systemLabels), @@ -2780,6 +2794,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), @@ -2798,6 +2813,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t1, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t1, @@ -2813,6 +2829,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule"], State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1, @@ -2844,9 +2861,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, LastSentAt: &t1, - Annotations: mergeLabels(baseRule.Annotations, data.Labels{ - "Error": genericError.Error(), - }), + Annotations: genericErrorAnnotations, }, }, }, @@ -2860,6 +2875,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: genericError, + Annotations: genericErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), @@ -2877,6 +2893,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule"], State: eval.Normal, StateReason: eval.Error.String(), + Annotations: genericErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1, @@ -2893,6 +2910,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule"], State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Annotations: genericErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1, @@ -2945,6 +2963,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Pending, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -2962,6 +2981,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, @@ -2978,6 +2998,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, @@ -2996,6 +3017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3014,6 +3036,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule + labels1"], State: eval.Normal, StateReason: eval.Error.String(), + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2, @@ -3031,6 +3054,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule + labels1"], State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3084,6 +3108,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3102,6 +3127,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, @@ -3118,6 +3144,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, @@ -3136,6 +3163,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3155,6 +3183,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t2, @@ -3172,6 +3201,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t2, @@ -3248,6 +3278,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: &State{ Labels: labels["system + rule"], Error: datasourceError, + Annotations: datasourceErrorAnnotations, State: eval.Normal, StateReason: ngmodels.StateReasonMissingSeries, LatestResult: newEvaluation(t1, eval.Error), @@ -3281,6 +3312,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.StateReasonMissingSeries, LatestResult: newEvaluation(t1, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t3, LastEvaluationTime: t3, @@ -3309,6 +3341,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.StateReasonMissingSeries, LatestResult: newEvaluation(t1, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t3, LastEvaluationTime: t3, @@ -3359,6 +3392,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3377,6 +3411,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t2, @@ -3393,6 +3428,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t2, @@ -3457,6 +3493,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3474,6 +3511,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule"], State: eval.Normal, StateReason: eval.Error.String(), + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2, @@ -3490,6 +3528,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Labels: labels["system + rule"], State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), @@ -3560,6 +3599,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Pending, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)? @@ -3592,6 +3632,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, @@ -3622,6 +3663,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Pending, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t2, @@ -3770,9 +3812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, LastSentAt: &t1, - Annotations: mergeLabels(baseRule.Annotations, data.Labels{ - "Error": genericError.Error(), - }), + Annotations: genericErrorAnnotations, }, }, }, @@ -3786,6 +3826,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: datasourceError, + Annotations: datasourceErrorAnnotations, LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), @@ -3803,6 +3844,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: genericError, + Annotations: genericErrorAnnotations, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), @@ -3821,12 +3863,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t1, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t1, }, }, }, + t2: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.Error.String(), + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: eval.Error.String(), + LatestResult: newEvaluation(t2, eval.Error), + Annotations: genericErrorAnnotations, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, }, ngmodels.KeepLastErrState: { t1: { @@ -3837,12 +3896,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t1, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t1, }, }, }, + t2: { + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + LatestResult: newEvaluation(t2, eval.Error), + Annotations: genericErrorAnnotations, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, }, }, }, diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 7905204cc71..b5ac2b51447 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -372,6 +372,8 @@ func TestProcessEvalResults(t *testing.T) { return r } + datasourceError := expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")) + labels1 := data.Labels{ "instance_label": "test-1", } @@ -392,6 +394,13 @@ func TestProcessEvalResults(t *testing.T) { "system + rule + no-data": mergeLabels(mergeLabels(noDataLabels, baseRule.Labels), systemLabels), } + datasourceErrorAnnotations := data.Labels{ + "annotation": "test", + "datasource_uid": "datasource_uid_1", + "ref_id": "A", + "Error": datasourceError.Error(), + } + // keep it separate to make code folding work correctly. type testCase struct { desc string @@ -1042,6 +1051,7 @@ func TestProcessEvalResults(t *testing.T) { State: eval.Pending, StateReason: eval.Error.String(), Error: errors.New("with_state_error"), + Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"}, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(state.ResendDelay * 4), @@ -1077,6 +1087,7 @@ func TestProcessEvalResults(t *testing.T) { State: eval.Alerting, StateReason: eval.Error.String(), Error: errors.New("with_state_error"), + Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"}, LatestResult: newEvaluation(tn(5), eval.Error), StartsAt: tn(5), EndsAt: tn(5).Add(state.ResendDelay * 4), @@ -1093,7 +1104,7 @@ func TestProcessEvalResults(t *testing.T) { newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), }, t2: { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, }, expectedAnnotations: 1, @@ -1109,7 +1120,7 @@ func TestProcessEvalResults(t *testing.T) { }), ResultFingerprint: labels1.Fingerprint(), State: eval.Error, - Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")), + Error: datasourceError, LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(state.ResendDelay * 4), @@ -1128,13 +1139,13 @@ func TestProcessEvalResults(t *testing.T) { newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), }, t2: { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, t3: { newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), }, tn(4): { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, }, expectedAnnotations: 1, @@ -1149,6 +1160,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: tn(4).Add(state.ResendDelay * 4), LastEvaluationTime: tn(4), LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3. + Annotations: datasourceErrorAnnotations, }, }, }, @@ -1163,10 +1175,10 @@ func TestProcessEvalResults(t *testing.T) { newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), }, t3: { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, tn(4): { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, }, expectedAnnotations: 2, @@ -1181,6 +1193,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: tn(4).Add(state.ResendDelay * 4), LastEvaluationTime: tn(4), LastSentAt: util.Pointer(tn(4)), + Annotations: datasourceErrorAnnotations, }, }, }, @@ -1192,7 +1205,7 @@ func TestProcessEvalResults(t *testing.T) { newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), }, t2: { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, }, expectedAnnotations: 1, @@ -1203,6 +1216,7 @@ func TestProcessEvalResults(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t1, EndsAt: t1, LastEvaluationTime: t2, @@ -1217,7 +1231,7 @@ func TestProcessEvalResults(t *testing.T) { newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), }, t2: { - newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different }, }, expectedAnnotations: 2, @@ -1228,6 +1242,7 @@ func TestProcessEvalResults(t *testing.T) { State: eval.Normal, StateReason: eval.Error.String(), LatestResult: newEvaluation(t2, eval.Error), + Annotations: datasourceErrorAnnotations, StartsAt: t2, EndsAt: t2, LastEvaluationTime: t2, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 57e7f36649d..e67effe8dcd 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -141,25 +141,38 @@ func (a *State) Maintain(interval int64, evaluatedAt time.Time) { a.EndsAt = nextEndsTime(interval, evaluatedAt) } -// AddErrorAnnotations adds annotations to the state to indicate that an error occurred. -func (a *State) AddErrorAnnotations(err error, rule *models.AlertRule) { +// AddErrorInformation adds annotations to the state to indicate that an error occurred. +// If addDatasourceInfoToLabels is true, the ref_id and datasource_uid are added to the labels, +// otherwise, they are added to the annotations. +func (a *State) AddErrorInformation(err error, rule *models.AlertRule, addDatasourceInfoToLabels bool) { if err == nil { return } a.Annotations["Error"] = err.Error() + // If the evaluation failed because a query returned an error then add the Ref ID and - // Datasource UID as labels + // Datasource UID as labels or annotations var utilError errutil.Error - if errors.As(a.Error, &utilError) && - (errors.Is(a.Error, expr.QueryError) || errors.Is(a.Error, expr.ConversionError)) { + if errors.As(err, &utilError) && + (errors.Is(err, expr.QueryError) || errors.Is(err, expr.ConversionError)) { for _, next := range rule.Data { if next.RefID == utilError.PublicPayload["refId"].(string) { - a.Labels["ref_id"] = next.RefID - a.Labels["datasource_uid"] = next.DatasourceUID + if addDatasourceInfoToLabels { + a.Labels["ref_id"] = next.RefID + a.Labels["datasource_uid"] = next.DatasourceUID + } else { + a.Annotations["ref_id"] = next.RefID + a.Annotations["datasource_uid"] = next.DatasourceUID + } break } } + } else { + // Remove the ref_id and datasource_uid from the annotations if they are present. + // It can happen if the alert state hasn't changed, but the error is different now. + delete(a.Annotations, "ref_id") + delete(a.Annotations, "datasource_uid") } } @@ -334,11 +347,12 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge resultAlerting(state, rule, result, logger, models.StateReasonError) // This is a special case where Alerting and Pending should also have an error and reason state.Error = result.Error + state.AddErrorInformation(result.Error, rule, false) case models.ErrorErrState: if state.State == eval.Error { prevEndsAt := state.EndsAt state.Error = result.Error - state.AddErrorAnnotations(result.Error, rule) + state.AddErrorInformation(result.Error, rule, true) state.Maintain(rule.IntervalSeconds, result.EvaluatedAt) logger.Debug("Keeping state", "state", @@ -360,18 +374,20 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge "next_ends_at", nextEndsAt) state.SetError(result.Error, result.EvaluatedAt, nextEndsAt) - state.AddErrorAnnotations(result.Error, rule) + state.AddErrorInformation(result.Error, rule, true) } case models.OkErrState: logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", handlerStr) resultNormal(state, rule, result, logger, "") // TODO: Should we add a reason? + state.AddErrorInformation(result.Error, rule, false) case models.KeepLastErrState: logger := logger.New("previous_handler", handlerStr) resultKeepLast(state, rule, result, logger) + state.AddErrorInformation(result.Error, rule, false) default: err := fmt.Errorf("unsupported execution error state: %s", rule.ExecErrState) state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)) - state.Annotations["Error"] = err.Error() + state.AddErrorInformation(result.Error, rule, false) } }