diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 8691417c8b7..c5210a70273 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -39,7 +39,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general- | `transformationsRedesign` | Enables the transformations redesign | Yes | | `traceQLStreaming` | Enables response streaming of TraceQL queries of the Tempo data source | | | `awsAsyncQueryCaching` | Enable caching for async queries for Redshift and Athena. Requires that the datasource has caching and async query support enabled | Yes | -| `alertingNoDataErrorExecution` | Changes how Alerting state manager handles execution of NoData/Error | Yes | | `angularDeprecationUI` | Display Angular warnings in dashboards and panels | Yes | | `dashgpt` | Enable AI powered features in dashboards | Yes | | `alertingInsights` | Show the new alerting insights landing page | Yes | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 39299b42166..033b7c30e05 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -86,7 +86,6 @@ export interface FeatureToggles { awsAsyncQueryCaching?: boolean; permissionsFilterRemoveSubquery?: boolean; configurableSchedulerTick?: boolean; - alertingNoDataErrorExecution?: boolean; angularDeprecationUI?: boolean; dashgpt?: boolean; aiGeneratedDashboardChanges?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index a05f3956e1b..1ef6f4bef32 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -508,15 +508,6 @@ var ( RequiresRestart: true, HideFromDocs: true, }, - { - Name: "alertingNoDataErrorExecution", - Description: "Changes how Alerting state manager handles execution of NoData/Error", - Stage: FeatureStageGeneralAvailability, - FrontendOnly: false, - Owner: grafanaAlertingSquad, - RequiresRestart: true, - Expression: "true", // enabled by default - }, { Name: "angularDeprecationUI", Description: "Display Angular warnings in dashboards and panels", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index c3e15a6d655..84efa8a1ad1 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -67,7 +67,6 @@ featureToggleAdminPage,experimental,@grafana/grafana-operator-experience-squad,f awsAsyncQueryCaching,GA,@grafana/aws-datasources,false,false,false permissionsFilterRemoveSubquery,experimental,@grafana/grafana-backend-group,false,false,false configurableSchedulerTick,experimental,@grafana/alerting-squad,false,true,false -alertingNoDataErrorExecution,GA,@grafana/alerting-squad,false,true,false angularDeprecationUI,GA,@grafana/plugins-platform-backend,false,false,true dashgpt,GA,@grafana/dashboards-squad,false,false,true aiGeneratedDashboardChanges,experimental,@grafana/dashboards-squad,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 67ed9660c83..3fcc06d9d94 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -279,10 +279,6 @@ const ( // Enable changing the scheduler base interval via configuration option unified_alerting.scheduler_tick_interval FlagConfigurableSchedulerTick = "configurableSchedulerTick" - // FlagAlertingNoDataErrorExecution - // Changes how Alerting state manager handles execution of NoData/Error - FlagAlertingNoDataErrorExecution = "alertingNoDataErrorExecution" - // FlagAngularDeprecationUI // Display Angular warnings in dashboards and panels FlagAngularDeprecationUI = "angularDeprecationUI" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 69911d2e36c..1564aeeec0b 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -342,6 +342,7 @@ "name": "alertingNoDataErrorExecution", "resourceVersion": "1720021873452", "creationTimestamp": "2023-08-15T14:27:15Z", + "deletionTimestamp": "2025-03-13T19:16:25Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC" } diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 690c58c291b..a01a3913752 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -409,20 +409,19 @@ func (ng *AlertNG) init() error { ng.InstanceStore, ng.StartupInstanceReader = initInstanceStore(ng.store.SQLStore, ng.Log, ng.FeatureToggles) stateManagerCfg := state.ManagerCfg{ - Metrics: ng.Metrics.GetStateMetrics(), - ExternalURL: appUrl, - DisableExecution: !ng.Cfg.UnifiedAlerting.ExecuteAlerts, - InstanceStore: ng.InstanceStore, - Images: ng.ImageService, - Clock: clk, - Historian: history, - ApplyNoDataAndErrorToAllStates: ng.FeatureToggles.IsEnabledGlobally(featuremgmt.FlagAlertingNoDataErrorExecution), - MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency, - StatePeriodicSaveBatchSize: ng.Cfg.UnifiedAlerting.StatePeriodicSaveBatchSize, - RulesPerRuleGroupLimit: ng.Cfg.UnifiedAlerting.RulesPerRuleGroupLimit, - Tracer: ng.tracer, - Log: log.New("ngalert.state.manager"), - ResolvedRetention: ng.Cfg.UnifiedAlerting.ResolvedAlertRetention, + Metrics: ng.Metrics.GetStateMetrics(), + ExternalURL: appUrl, + DisableExecution: !ng.Cfg.UnifiedAlerting.ExecuteAlerts, + InstanceStore: ng.InstanceStore, + Images: ng.ImageService, + Clock: clk, + Historian: history, + MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency, + StatePeriodicSaveBatchSize: ng.Cfg.UnifiedAlerting.StatePeriodicSaveBatchSize, + RulesPerRuleGroupLimit: ng.Cfg.UnifiedAlerting.RulesPerRuleGroupLimit, + Tracer: ng.tracer, + Log: log.New("ngalert.state.manager"), + ResolvedRetention: ng.Cfg.UnifiedAlerting.ResolvedAlertRetention, } statePersister := initStatePersister(ng.Cfg.UnifiedAlerting, stateManagerCfg, ng.FeatureToggles) stateManager := state.NewManager(stateManagerCfg, statePersister) diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index e23d3bce143..d45290b887a 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -55,8 +55,7 @@ type Manager struct { historian Historian externalURL *url.URL - applyNoDataAndErrorToAllStates bool - rulesPerRuleGroupLimit int64 + rulesPerRuleGroupLimit int64 persister StatePersister } @@ -73,10 +72,8 @@ type ManagerCfg struct { // StatePeriodicSaveBatchSize controls the size of the alert instance batch that is saved periodically when the // alertingSaveStatePeriodic feature flag is enabled. StatePeriodicSaveBatchSize int - // ApplyNoDataAndErrorToAllStates makes state manager to apply exceptional results (NoData and Error) - // to all states when corresponding execution in the rule definition is set to either `Alerting` or `OK` - ApplyNoDataAndErrorToAllStates bool - RulesPerRuleGroupLimit int64 + + RulesPerRuleGroupLimit int64 DisableExecution bool @@ -96,24 +93,19 @@ func NewManager(cfg ManagerCfg, statePersister StatePersister) *Manager { } m := &Manager{ - cache: c, - ResendDelay: ResendDelay, // TODO: make this configurable - ResolvedRetention: cfg.ResolvedRetention, - log: cfg.Log, - metrics: cfg.Metrics, - instanceStore: cfg.InstanceStore, - images: cfg.Images, - historian: cfg.Historian, - clock: cfg.Clock, - externalURL: cfg.ExternalURL, - applyNoDataAndErrorToAllStates: cfg.ApplyNoDataAndErrorToAllStates, - rulesPerRuleGroupLimit: cfg.RulesPerRuleGroupLimit, - persister: statePersister, - tracer: cfg.Tracer, - } - - if m.applyNoDataAndErrorToAllStates { - m.log.Info("Running in alternative execution of Error/NoData mode") + cache: c, + ResendDelay: ResendDelay, // TODO: make this configurable + ResolvedRetention: cfg.ResolvedRetention, + log: cfg.Log, + metrics: cfg.Metrics, + instanceStore: cfg.InstanceStore, + images: cfg.Images, + historian: cfg.Historian, + clock: cfg.Clock, + externalURL: cfg.ExternalURL, + rulesPerRuleGroupLimit: cfg.RulesPerRuleGroupLimit, + persister: statePersister, + tracer: cfg.Tracer, } return m @@ -358,7 +350,7 @@ func (st *Manager) ProcessEvalResults( } logger.Debug("State manager processing evaluation results", "resultCount", len(results)) - states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger, fn) + states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger, fn, evaluatedAt) staleStates := st.deleteStaleStatesFromCache(logger, evaluatedAt, alertRule, fn) span.AddEvent("results processed", trace.WithAttributes( @@ -402,8 +394,8 @@ func (st *Manager) updateLastSentAt(states StateTransitions, evaluatedAt time.Ti return result } -func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger, takeImageFn takeImageFn) []StateTransition { - if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state +func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger, takeImageFn takeImageFn, now time.Time) []StateTransition { + if results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state // aggregate UID of datasources that returned NoData into one and provide as auxiliary info via annotationa. See: https://github.com/grafana/grafana/issues/88184 var refIds strings.Builder var datasourceUIDs strings.Builder @@ -430,12 +422,20 @@ func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels. "datasource_uid": datasourceUIDs.String(), "ref_id": refIds.String(), } - transitions := st.setNextStateForAll(alertRule, results[0], logger, annotations, takeImageFn) + result := eval.Result{ + Instance: data.Labels{}, + State: eval.NoData, + EvaluatedAt: now, + } + if len(results) > 0 { + result = results[0] + } + transitions := st.setNextStateForAll(alertRule, result, logger, annotations, takeImageFn) if len(transitions) > 0 { return transitions // if there are no current states for the rule. Create ones for each result } } - if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) { + if results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) { // TODO squash all errors into one, and provide as annotation transitions := st.setNextStateForAll(alertRule, results[0], logger, nil, takeImageFn) if len(transitions) > 0 { diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 42fcddf41a3..0c4a9d6635c 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -293,7 +293,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { } } - executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) { + executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition) { clk := clock.NewMock() testMetrics := metrics.NewNGAlert(prometheus.NewPedanticRegistry()).GetStateMetrics() @@ -306,8 +306,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Images: &NotAvailableImageService{}, Clock: clk, Historian: &FakeHistorian{}, - - ApplyNoDataAndErrorToAllStates: applyNoDataErrorToAllStates, } st := NewManager(cfg, NewNoopPersister()) @@ -1114,12 +1112,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - t.Run("applyNoDataErrorToAllStates=true", func(t *testing.T) { - executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, true) - }) - t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { - executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, false) - }) + executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions) }) } @@ -1136,8 +1129,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ruleMutators []ngmodels.AlertRuleMutator results map[time.Time]eval.Results expectedTransitions map[ngmodels.NoDataState]map[time.Time][]StateTransition - - expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.NoDataState]map[time.Time][]StateTransition } executeForEachRule := func(t *testing.T, tc noDataTestCase) { @@ -1148,25 +1139,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { r = ngmodels.CopyRule(r, tc.ruleMutators...) } t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec] - overridden := "[*]" - if !ok { - expectedTransitions, ok = tc.expectedTransitions[stateExec] - overridden = "" - } + expectedTransitions, ok := tc.expectedTransitions[stateExec] if !ok { require.Fail(t, "no expected state transitions") } - t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) { - executeTest(t, r, tc.results, expectedTransitions, true) - }) - t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitions[stateExec] - if !ok { - require.Fail(t, "no expected state transitions") - } - executeTest(t, r, tc.results, expectedTransitions, false) - }) + executeTest(t, r, tc.results, expectedTransitions) }) } } @@ -1279,59 +1256,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{}), - StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 4), - LastEvaluationTime: t2, - LastSentAt: &t2, - Values: map[string]float64{}, - }, - }, - }, - }, - ngmodels.OK: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{}), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - Values: map[string]float64{}, - }, - }, - }, - }, - ngmodels.KeepLast: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -1460,138 +1384,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Alerting, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t2, - }, - }, - }, - }, - ngmodels.OK: { - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - ngmodels.KeepLast: { - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -1854,146 +1646,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Pending, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 4), - LastEvaluationTime: t2, - }, - }, - }, - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Pending, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - ngmodels.OK: { - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - ngmodels.KeepLast: { - t3: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Normal), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels2"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -2212,56 +1864,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - ngmodels.OK: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - ngmodels.KeepLast: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule + labels1"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t3: { { @@ -2471,57 +2073,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 4), - LastEvaluationTime: t2, - LastSentAt: &t2, - }, - }, - }, - }, - ngmodels.OK: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - ngmodels.KeepLast: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -2634,102 +2185,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t3: { - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Alerting, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t2, - }, - }, - }, - }, - ngmodels.OK: { - t3: { - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: eval.NoData.String(), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - ngmodels.KeepLast: { - t3: { - { - PreviousState: eval.Alerting, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: ngmodels.StateReasonMissingSeries, - LatestResult: newEvaluation(t1, eval.Alerting), - StartsAt: t1, - EndsAt: t3, - LastEvaluationTime: t3, - ResolvedAt: &t3, - LastSentAt: &t3, - }, - }, - { - PreviousState: eval.Normal, - PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - State: &State{ - Labels: labels["system + rule + no-data"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t3, eval.NoData), - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t3, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -2884,56 +2339,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.Alerting: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - ngmodels.OK: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - ngmodels.KeepLast: { - t3: { - { - PreviousState: eval.Pending, - State: &State{ - Labels: labels["system + rule"], - State: eval.Alerting, - LatestResult: newEvaluation(t3, eval.Alerting), - StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 4), - LastEvaluationTime: t3, - LastSentAt: &t3, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { t2: { { @@ -3053,8 +2458,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ruleMutators []ngmodels.AlertRuleMutator results map[time.Time]eval.Results expectedTransitions map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition - - expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition } executeForEachRule := func(t *testing.T, tc errorTestCase) { @@ -3065,25 +2468,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { r = ngmodels.CopyRule(r, tc.ruleMutators...) } t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec] - overridden := "[*]" - if !ok { - expectedTransitions, ok = tc.expectedTransitions[stateExec] - overridden = "" - } + expectedTransitions, ok := tc.expectedTransitions[stateExec] if !ok { require.Fail(t, "no expected state transitions") } - t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) { - executeTest(t, r, tc.results, expectedTransitions, true) - }) - t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitions[stateExec] - if !ok { - require.Fail(t, "no expected state transitions") - } - executeTest(t, r, tc.results, expectedTransitions, false) - }) + executeTest(t, r, tc.results, expectedTransitions) }) } } @@ -3287,60 +2676,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.AlertingErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Pending, - StateReason: eval.Error.String(), - Error: datasourceError, - Annotations: datasourceErrorAnnotations, - LatestResult: newEvaluation(t2, eval.Error), - StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 4), - LastEvaluationTime: t2, - }, - }, - }, - }, - ngmodels.OkErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: eval.Error.String(), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - ngmodels.KeepLastErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.AlertingErrState: { t2: { { @@ -3432,61 +2767,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, - ngmodels.AlertingErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Alerting, - StateReason: eval.Error.String(), - Error: datasourceError, - Annotations: datasourceErrorAnnotations, - LatestResult: newEvaluation(t2, eval.Error), - StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 4), - LastEvaluationTime: t2, - LastSentAt: &t2, - }, - }, - }, - }, - ngmodels.OkErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: eval.Error.String(), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - ngmodels.KeepLastErrState: { - t2: { - { - PreviousState: eval.Normal, - State: &State{ - Labels: labels["system + rule"], - State: eval.Normal, - StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - }, - }, - expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.AlertingErrState: { t2: { { diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 1af185871b8..030a360b86c 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -379,8 +379,6 @@ func TestProcessEvalResults(t *testing.T) { return r } - datasourceError := expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")) - labels1 := data.Labels{ "instance_label": "test-1", } @@ -401,13 +399,6 @@ func TestProcessEvalResults(t *testing.T) { "system + rule + no-data": mergeLabels(mergeLabels(noDataLabels, baseRule.Labels), systemLabels), } - datasourceErrorAnnotations := data.Labels{ - "annotation": "test", - "datasource_uid": "datasource_uid_1", - "ref_id": "A", - "Error": datasourceError.Error(), - } - // keep it separate to make code folding work correctly. type testCase struct { desc string @@ -750,66 +741,6 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, - { - desc: "normal -> pending when For is set but not exceeded, result is NoData and NoDataState is alerting", - alertRule: baseRuleWith(m.WithForNTimes(6), m.WithNoDataExecAs(models.Alerting)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), - }, - }, - expectedAnnotations: 1, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Pending, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 4), - LastEvaluationTime: t2, - }, - }, - }, - { - desc: "normal -> alerting when For is exceeded, result is NoData and NoDataState is alerting", - alertRule: baseRuleWith(m.WithForNTimes(3), m.WithNoDataExecAs(models.Alerting)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because nodata has no labels of regular result - }, - t3: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), - }, - tn(5): { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), - }, - }, - expectedAnnotations: 2, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Alerting, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(tn(5), eval.NoData), - StartsAt: tn(5), - EndsAt: tn(5).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(5), - LastSentAt: util.Pointer(tn(5)), - }, - }, - }, { desc: "normal -> nodata when result is NoData and NoDataState is nodata", alertRule: baseRule, @@ -950,95 +881,6 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, - { - desc: "normal -> normal (NoData, KeepLastState) -> alerting -> alerting (NoData, KeepLastState) - keeps last state when result is NoData and NoDataState is KeepLast", - alertRule: baseRuleWith(m.WithForNTimes(0), m.WithNoDataExecAs(models.KeepLast)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels - }, - t3: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels - }, - }, - expectedAnnotations: 1, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Alerting, - StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), - LatestResult: newEvaluation(tn(4), eval.NoData), - StartsAt: t3, - EndsAt: tn(4).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(4), - LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3. - }, - }, - }, - { - desc: "normal -> pending -> pending (NoData, KeepLastState) -> alerting (NoData, KeepLastState) - keep last state respects For when result is NoData", - alertRule: baseRuleWith(m.WithForNTimes(2), m.WithNoDataExecAs(models.KeepLast)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - t3: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels - }, - tn(4): { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels - }, - }, - expectedAnnotations: 2, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Alerting, - StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), - LatestResult: newEvaluation(tn(4), eval.NoData), - StartsAt: tn(4), - EndsAt: tn(4).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(4), - LastSentAt: util.Pointer(tn(4)), - }, - }, - }, - { - desc: "normal -> normal when result is NoData and NoDataState is ok", - alertRule: baseRuleWith(m.WithNoDataExecAs(models.OK)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels - }, - }, - expectedAnnotations: 0, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Normal, - StateReason: eval.NoData.String(), - LatestResult: newEvaluation(t2, eval.NoData), - StartsAt: t1, - EndsAt: t1, - LastEvaluationTime: t2, - }, - }, - }, { desc: "normal -> pending when For is set but not exceeded, result is Error and ExecErrState is Alerting", alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.AlertingErrState)), @@ -1103,260 +945,6 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, - { - desc: "normal -> error when result is Error and ExecErrState is Error", - alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.ErrorErrState)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - }, - expectedAnnotations: 1, - expectedStates: []*state.State{ - { - CacheID: func() data.Fingerprint { - lbls := models.InstanceLabels(labels["system + rule + labels1"]) - return lbls.Fingerprint() - }(), - Labels: mergeLabels(labels["system + rule + labels1"], data.Labels{ - "datasource_uid": "datasource_uid_1", - "ref_id": "A", - }), - ResultFingerprint: labels1.Fingerprint(), - State: eval.Error, - Error: datasourceError, - LatestResult: newEvaluation(t2, eval.Error), - StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 4), - LastEvaluationTime: t2, - LastSentAt: &t2, - EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "Error": "[sse.dataQueryError] failed to execute query [A]: this is an error"}, - }, - }, - }, - { - desc: "normal -> normal (Error, KeepLastState) -> alerting -> alerting (Error, KeepLastState) - keeps last state when result is Error and ExecErrState is KeepLast", - alertRule: baseRuleWith(m.WithForNTimes(0), m.WithErrorExecAs(models.KeepLastErrState)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - t3: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - }, - expectedAnnotations: 1, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Alerting, - StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), - LatestResult: newEvaluation(tn(4), eval.Error), - StartsAt: t3, - EndsAt: tn(4).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(4), - LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3. - Annotations: datasourceErrorAnnotations, - }, - }, - }, - { - desc: "normal -> pending -> pending (Error, KeepLastState) -> alerting (Error, KeepLastState) - keep last state respects For when result is Error", - alertRule: baseRuleWith(m.WithForNTimes(2), m.WithErrorExecAs(models.KeepLastErrState)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - t3: { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - tn(4): { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - }, - expectedAnnotations: 2, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Alerting, - StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), - LatestResult: newEvaluation(tn(4), eval.Error), - StartsAt: tn(4), - EndsAt: tn(4).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(4), - LastSentAt: util.Pointer(tn(4)), - Annotations: datasourceErrorAnnotations, - }, - }, - }, - { - desc: "normal -> normal when result is Error and ExecErrState is OK", - alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.OkErrState)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - }, - expectedAnnotations: 1, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Normal, - StateReason: eval.Error.String(), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t1, - EndsAt: t1, - LastEvaluationTime: t2, - }, - }, - }, - { - desc: "alerting -> normal when result is Error and ExecErrState is OK", - alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.OkErrState)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different - }, - }, - expectedAnnotations: 2, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Normal, - StateReason: eval.Error.String(), - LatestResult: newEvaluation(t2, eval.Error), - Annotations: datasourceErrorAnnotations, - StartsAt: t2, - EndsAt: t2, - LastEvaluationTime: t2, - }, - }, - }, - { - desc: "normal -> alerting -> error when result is Error and ExecErrorState is Error", - alertRule: baseRuleWith(m.WithForNTimes(2)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - t2: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - t3: { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created - }, - tn(5): { - newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created - }, - tn(6): { - newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created - }, - }, - expectedAnnotations: 3, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Error, - Error: fmt.Errorf("with_state_error"), - LatestResult: newEvaluation(tn(6), eval.Error), - StartsAt: tn(4), - EndsAt: tn(6).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(6), - LastSentAt: util.Pointer(tn(6)), // After 30s resend delay, last sent at is t6. - Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"}, - }, - }, - }, - { - desc: "normal -> alerting -> error -> alerting - it should clear the error", - alertRule: baseRuleWith(m.WithForNTimes(3)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - tn(5): { - newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO fix it - }, - tn(8): { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - }, - expectedAnnotations: 3, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.Pending, - LatestResult: newEvaluation(tn(8), eval.Alerting), - StartsAt: tn(8), - EndsAt: tn(8).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(8), - LastSentAt: util.Pointer(tn(5)), - }, - }, - }, - { - desc: "normal -> alerting -> error -> no data - it should clear the error", - alertRule: baseRuleWith(m.WithForNTimes(3)), - evalResults: map[time.Time]eval.Results{ - t1: { - newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), - }, - tn(4): { - newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), - }, - tn(5): { - newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO FIX it - }, - tn(6): { - newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because it's not possible - }, - }, - expectedAnnotations: 3, - expectedStates: []*state.State{ - { - Labels: labels["system + rule + labels1"], - ResultFingerprint: labels1.Fingerprint(), - State: eval.NoData, - LatestResult: newEvaluation(tn(6), eval.NoData), - StartsAt: tn(6), - EndsAt: tn(6).Add(state.ResendDelay * 4), - LastEvaluationTime: tn(6), - LastSentAt: util.Pointer(tn(5)), - }, - }, - }, { desc: "template is correctly expanded", alertRule: baseRuleWith(