diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 35aeff3c4d7..7131f0165e0 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -4,6 +4,7 @@ import ( "context" "net/url" "strconv" + "strings" "time" "github.com/benbjohnson/clock" @@ -369,9 +370,34 @@ func (st *Manager) updateLastSentAt(states StateTransitions, evaluatedAt time.Ti func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition { if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state - // TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation + // aggregate UID of datasources that returned NoData into one and provide as auxiliary info via annotationa. See: https://github.com/grafana/grafana/issues/88184 + var refIds strings.Builder + var datasourceUIDs strings.Builder + // for deduplication of datasourceUIDs + dsUIDSet := make(map[string]bool) + for i, result := range results { + if refid, ok := result.Instance["ref_id"]; ok { + if i > 0 { + refIds.WriteString(",") + } + refIds.WriteString(refid) + } + if dsUID, ok := result.Instance["datasource_uid"]; ok { + if !dsUIDSet[dsUID] { + if i > 0 { + refIds.WriteString(",") + } + datasourceUIDs.WriteString(dsUID) + dsUIDSet[dsUID] = true + } + } + } transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger) if len(transitions) > 0 { + for _, t := range transitions { + t.State.Annotations["datasource_uid"] = datasourceUIDs.String() + t.State.Annotations["ref_id"] = refIds.String() + } return transitions // if there are no current states for the rule. Create ones for each result } } diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index e263d11155d..254200f100c 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -197,6 +197,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { "ref_id": "A", } + noDataAnnotations := data.Labels{ + "datasource_uid": "1", + "ref_id": "A", + } + labels := map[string]data.Labels{ "system + rule": mergeLabels(baseRule.Labels, systemLabels), "system + rule + labels1": mergeLabels(mergeLabels(labels1, baseRule.Labels), systemLabels), @@ -986,6 +991,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}), @@ -1004,6 +1010,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}), @@ -1021,6 +1028,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}), @@ -1243,6 +1251,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1256,6 +1265,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1273,6 +1283,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: &State{ Labels: labels["system + rule + labels1"], State: eval.Alerting, + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, @@ -1287,6 +1298,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: &State{ Labels: labels["system + rule + labels2"], State: eval.Alerting, + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, @@ -1303,6 +1315,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1315,6 +1328,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1332,6 +1346,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1345,6 +1360,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1363,6 +1379,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -1375,6 +1392,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -1391,6 +1409,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t3, eval.NoData), @@ -1404,6 +1423,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t3, eval.NoData), @@ -1633,6 +1653,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Pending, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1645,6 +1666,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Pending, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1661,6 +1683,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1675,6 +1698,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1692,6 +1716,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1704,6 +1729,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Pending, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -1719,6 +1745,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1732,6 +1759,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -1748,6 +1776,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -1760,6 +1789,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Pending, State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -1776,6 +1806,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), State: &State{ Labels: labels["system + rule + labels1"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t3, eval.NoData), @@ -1789,6 +1820,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), State: &State{ Labels: labels["system + rule + labels2"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t3, eval.NoData), @@ -2149,6 +2181,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -2166,6 +2199,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -2182,6 +2216,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Normal, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -2354,6 +2389,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -2370,6 +2406,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -2387,6 +2424,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t2, eval.NoData), @@ -2404,6 +2442,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: eval.NoData.String(), State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Normal, StateReason: eval.NoData.String(), LatestResult: newEvaluation(t3, eval.NoData), @@ -2422,6 +2461,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousState: eval.Alerting, State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t2, eval.NoData), @@ -2438,6 +2478,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), State: &State{ Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), State: eval.Alerting, StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), LatestResult: newEvaluation(t3, eval.NoData), @@ -2452,7 +2493,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, { - desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2*,t3", + desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2,t3", ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.RuleMuts.WithForNTimes(2)}, results: map[time.Time]eval.Results{ t1: { @@ -2547,6 +2588,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), + State: eval.Pending, + StateReason: eval.NoData.String(), + LatestResult: newEvaluation(t2, eval.NoData), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Pending, @@ -2564,6 +2620,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, ngmodels.OK: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), + State: eval.Normal, + StateReason: eval.NoData.String(), + LatestResult: newEvaluation(t2, eval.NoData), + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Normal, @@ -2580,6 +2651,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), + State: eval.Pending, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + LatestResult: newEvaluation(t2, eval.NoData), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Pending,