public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Remove feature flag alertingNoDataErrorExecution (#102156)

Browse Source 
* remove feature flag

* remove feature flag in state manager

* make sure no data with empty results is handled

Signed-off-by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>

---------

Signed-off-by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>
This commit is contained in:
Yuri Tseretyan
2025-03-14 14:51:58 -04:00
committed by GitHub
parent f1f544ec5b
commit e30034a42a
10 changed files with 49 additions and 1197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
-1
View File
@@ -39,7 +39,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
| `transformationsRedesign` | Enables the transformations redesign | Yes |
| `traceQLStreaming` | Enables response streaming of TraceQL queries of the Tempo data source | |
| `awsAsyncQueryCaching` | Enable caching for async queries for Redshift and Athena. Requires that the datasource has caching and async query support enabled | Yes |
| `alertingNoDataErrorExecution` | Changes how Alerting state manager handles execution of NoData/Error | Yes |
| `angularDeprecationUI` | Display Angular warnings in dashboards and panels | Yes |
| `dashgpt` | Enable AI powered features in dashboards | Yes |
| `alertingInsights` | Show the new alerting insights landing page | Yes |
packages/grafana-data/src/types/featureToggles.gen.ts
-1
View File
@@ -86,7 +86,6 @@ export interface FeatureToggles {
awsAsyncQueryCaching?: boolean;
permissionsFilterRemoveSubquery?: boolean;
configurableSchedulerTick?: boolean;
alertingNoDataErrorExecution?: boolean;
angularDeprecationUI?: boolean;
dashgpt?: boolean;
aiGeneratedDashboardChanges?: boolean;
pkg/services/featuremgmt/registry.go
-9
View File
@@ -508,15 +508,6 @@ var (
RequiresRestart: true,
HideFromDocs: true,
},
{
Name: "alertingNoDataErrorExecution",
Description: "Changes how Alerting state manager handles execution of NoData/Error",
Stage: FeatureStageGeneralAvailability,
FrontendOnly: false,
Owner: grafanaAlertingSquad,
RequiresRestart: true,
Expression: "true", // enabled by default
},
{
Name: "angularDeprecationUI",
Description: "Display Angular warnings in dashboards and panels",
pkg/services/featuremgmt/toggles_gen.csv
-1
View File
@@ -67,7 +67,6 @@ featureToggleAdminPage,experimental,@grafana/grafana-operator-experience-squad,f
awsAsyncQueryCaching,GA,@grafana/aws-datasources,false,false,false
permissionsFilterRemoveSubquery,experimental,@grafana/grafana-backend-group,false,false,false
configurableSchedulerTick,experimental,@grafana/alerting-squad,false,true,false
alertingNoDataErrorExecution,GA,@grafana/alerting-squad,false,true,false
angularDeprecationUI,GA,@grafana/plugins-platform-backend,false,false,true
dashgpt,GA,@grafana/dashboards-squad,false,false,true
aiGeneratedDashboardChanges,experimental,@grafana/dashboards-squad,false,false,true
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
67 awsAsyncQueryCaching GA @grafana/aws-datasources false false false
68 permissionsFilterRemoveSubquery experimental @grafana/grafana-backend-group false false false
69 configurableSchedulerTick experimental @grafana/alerting-squad false true false
alertingNoDataErrorExecution GA @grafana/alerting-squad false true false
70 angularDeprecationUI GA @grafana/plugins-platform-backend false false true
71 dashgpt GA @grafana/dashboards-squad false false true
72 aiGeneratedDashboardChanges experimental @grafana/dashboards-squad false false true
pkg/services/featuremgmt/toggles_gen.go
-4
View File
@@ -279,10 +279,6 @@ const (
// Enable changing the scheduler base interval via configuration option unified_alerting.scheduler_tick_interval
FlagConfigurableSchedulerTick = "configurableSchedulerTick"
// FlagAlertingNoDataErrorExecution
// Changes how Alerting state manager handles execution of NoData/Error
FlagAlertingNoDataErrorExecution = "alertingNoDataErrorExecution"
// FlagAngularDeprecationUI
// Display Angular warnings in dashboards and panels
FlagAngularDeprecationUI = "angularDeprecationUI"
pkg/services/featuremgmt/toggles_gen.json
+1
View File
@@ -342,6 +342,7 @@
"name": "alertingNoDataErrorExecution",
"resourceVersion": "1720021873452",
"creationTimestamp": "2023-08-15T14:27:15Z",
"deletionTimestamp": "2025-03-13T19:16:25Z",
"annotations": {
"grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC"
}
pkg/services/ngalert/ngalert.go
+13 -14
View File
@@ -409,20 +409,19 @@ func (ng *AlertNG) init() error {
ng.InstanceStore, ng.StartupInstanceReader = initInstanceStore(ng.store.SQLStore, ng.Log, ng.FeatureToggles)
stateManagerCfg := state.ManagerCfg{
Metrics: ng.Metrics.GetStateMetrics(),
ExternalURL: appUrl,
DisableExecution: !ng.Cfg.UnifiedAlerting.ExecuteAlerts,
InstanceStore: ng.InstanceStore,
Images: ng.ImageService,
Clock: clk,
Historian: history,
ApplyNoDataAndErrorToAllStates: ng.FeatureToggles.IsEnabledGlobally(featuremgmt.FlagAlertingNoDataErrorExecution),
MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency,
StatePeriodicSaveBatchSize: ng.Cfg.UnifiedAlerting.StatePeriodicSaveBatchSize,
RulesPerRuleGroupLimit: ng.Cfg.UnifiedAlerting.RulesPerRuleGroupLimit,
Tracer: ng.tracer,
Log: log.New("ngalert.state.manager"),
ResolvedRetention: ng.Cfg.UnifiedAlerting.ResolvedAlertRetention,
Metrics: ng.Metrics.GetStateMetrics(),
ExternalURL: appUrl,
DisableExecution: !ng.Cfg.UnifiedAlerting.ExecuteAlerts,
InstanceStore: ng.InstanceStore,
Images: ng.ImageService,
Clock: clk,
Historian: history,
MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency,
StatePeriodicSaveBatchSize: ng.Cfg.UnifiedAlerting.StatePeriodicSaveBatchSize,
RulesPerRuleGroupLimit: ng.Cfg.UnifiedAlerting.RulesPerRuleGroupLimit,
Tracer: ng.tracer,
Log: log.New("ngalert.state.manager"),
ResolvedRetention: ng.Cfg.UnifiedAlerting.ResolvedAlertRetention,
}
statePersister := initStatePersister(ng.Cfg.UnifiedAlerting, stateManagerCfg, ng.FeatureToggles)
stateManager := state.NewManager(stateManagerCfg, statePersister)
pkg/services/ngalert/state/manager.go
+29 -29
View File
@@ -55,8 +55,7 @@ type Manager struct {
historian Historian
externalURL *url.URL
applyNoDataAndErrorToAllStates bool
rulesPerRuleGroupLimit int64
rulesPerRuleGroupLimit int64
persister StatePersister
}
@@ -73,10 +72,8 @@ type ManagerCfg struct {
// StatePeriodicSaveBatchSize controls the size of the alert instance batch that is saved periodically when the
// alertingSaveStatePeriodic feature flag is enabled.
StatePeriodicSaveBatchSize int
// ApplyNoDataAndErrorToAllStates makes state manager to apply exceptional results (NoData and Error)
// to all states when corresponding execution in the rule definition is set to either `Alerting` or `OK`
ApplyNoDataAndErrorToAllStates bool
RulesPerRuleGroupLimit int64
RulesPerRuleGroupLimit int64
DisableExecution bool
@@ -96,24 +93,19 @@ func NewManager(cfg ManagerCfg, statePersister StatePersister) *Manager {
}
m := &Manager{
cache: c,
ResendDelay: ResendDelay, // TODO: make this configurable
ResolvedRetention: cfg.ResolvedRetention,
log: cfg.Log,
metrics: cfg.Metrics,
instanceStore: cfg.InstanceStore,
images: cfg.Images,
historian: cfg.Historian,
clock: cfg.Clock,
externalURL: cfg.ExternalURL,
applyNoDataAndErrorToAllStates: cfg.ApplyNoDataAndErrorToAllStates,
rulesPerRuleGroupLimit: cfg.RulesPerRuleGroupLimit,
persister: statePersister,
tracer: cfg.Tracer,
}
if m.applyNoDataAndErrorToAllStates {
m.log.Info("Running in alternative execution of Error/NoData mode")
cache: c,
ResendDelay: ResendDelay, // TODO: make this configurable
ResolvedRetention: cfg.ResolvedRetention,
log: cfg.Log,
metrics: cfg.Metrics,
instanceStore: cfg.InstanceStore,
images: cfg.Images,
historian: cfg.Historian,
clock: cfg.Clock,
externalURL: cfg.ExternalURL,
rulesPerRuleGroupLimit: cfg.RulesPerRuleGroupLimit,
persister: statePersister,
tracer: cfg.Tracer,
}
return m
@@ -358,7 +350,7 @@ func (st *Manager) ProcessEvalResults(
}
logger.Debug("State manager processing evaluation results", "resultCount", len(results))
states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger, fn)
states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger, fn, evaluatedAt)
staleStates := st.deleteStaleStatesFromCache(logger, evaluatedAt, alertRule, fn)
span.AddEvent("results processed", trace.WithAttributes(
@@ -402,8 +394,8 @@ func (st *Manager) updateLastSentAt(states StateTransitions, evaluatedAt time.Ti
return result
}
func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger, takeImageFn takeImageFn) []StateTransition {
if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state
func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger, takeImageFn takeImageFn, now time.Time) []StateTransition {
if results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state
// aggregate UID of datasources that returned NoData into one and provide as auxiliary info via annotationa. See: https://github.com/grafana/grafana/issues/88184
var refIds strings.Builder
var datasourceUIDs strings.Builder
@@ -430,12 +422,20 @@ func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.
"datasource_uid": datasourceUIDs.String(),
"ref_id": refIds.String(),
}
transitions := st.setNextStateForAll(alertRule, results[0], logger, annotations, takeImageFn)
result := eval.Result{
Instance: data.Labels{},
State: eval.NoData,
EvaluatedAt: now,
}
if len(results) > 0 {
result = results[0]
}
transitions := st.setNextStateForAll(alertRule, result, logger, annotations, takeImageFn)
if len(transitions) > 0 {
return transitions // if there are no current states for the rule. Create ones for each result
}
}
if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) {
if results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) {
// TODO squash all errors into one, and provide as annotation
transitions := st.setNextStateForAll(alertRule, results[0], logger, nil, takeImageFn)
if len(transitions) > 0 {
pkg/services/ngalert/state/manager_private_test.go
+6 -726
View File
@@ -293,7 +293,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
}
}
executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) {
executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition) {
clk := clock.NewMock()
testMetrics := metrics.NewNGAlert(prometheus.NewPedanticRegistry()).GetStateMetrics()
@@ -306,8 +306,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
Images: &NotAvailableImageService{},
Clock: clk,
Historian: &FakeHistorian{},
ApplyNoDataAndErrorToAllStates: applyNoDataErrorToAllStates,
}
st := NewManager(cfg, NewNoopPersister())
@@ -1114,12 +1112,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
t.Run("applyNoDataErrorToAllStates=true", func(t *testing.T) {
executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, true)
})
t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, false)
})
executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions)
})
}
@@ -1136,8 +1129,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
ruleMutators []ngmodels.AlertRuleMutator
results map[time.Time]eval.Results
expectedTransitions map[ngmodels.NoDataState]map[time.Time][]StateTransition
expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.NoDataState]map[time.Time][]StateTransition
}
executeForEachRule := func(t *testing.T, tc noDataTestCase) {
@@ -1148,25 +1139,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
r = ngmodels.CopyRule(r, tc.ruleMutators...)
}
t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec]
overridden := "[*]"
if !ok {
expectedTransitions, ok = tc.expectedTransitions[stateExec]
overridden = ""
}
expectedTransitions, ok := tc.expectedTransitions[stateExec]
if !ok {
require.Fail(t, "no expected state transitions")
}
t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) {
executeTest(t, r, tc.results, expectedTransitions, true)
})
t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
expectedTransitions, ok := tc.expectedTransitions[stateExec]
if !ok {
require.Fail(t, "no expected state transitions")
}
executeTest(t, r, tc.results, expectedTransitions, false)
})
executeTest(t, r, tc.results, expectedTransitions)
})
}
}
@@ -1279,59 +1256,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{}),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
Values: map[string]float64{},
},
},
},
},
ngmodels.OK: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{}),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
Values: map[string]float64{},
},
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -1460,138 +1384,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t2,
},
},
},
},
ngmodels.OK: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -1854,146 +1646,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Pending,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.OK: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -2212,56 +1864,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.OK: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t3: {
{
@@ -2471,57 +2073,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
},
},
},
},
ngmodels.OK: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -2634,102 +2185,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t3: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Alerting,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t2,
},
},
},
},
ngmodels.OK: {
t3: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -2884,56 +2339,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.Alerting: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.OK: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
LastSentAt: &t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
@@ -3053,8 +2458,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
ruleMutators []ngmodels.AlertRuleMutator
results map[time.Time]eval.Results
expectedTransitions map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition
expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition
}
executeForEachRule := func(t *testing.T, tc errorTestCase) {
@@ -3065,25 +2468,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
r = ngmodels.CopyRule(r, tc.ruleMutators...)
}
t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec]
overridden := "[*]"
if !ok {
expectedTransitions, ok = tc.expectedTransitions[stateExec]
overridden = ""
}
expectedTransitions, ok := tc.expectedTransitions[stateExec]
if !ok {
require.Fail(t, "no expected state transitions")
}
t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) {
executeTest(t, r, tc.results, expectedTransitions, true)
})
t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
expectedTransitions, ok := tc.expectedTransitions[stateExec]
if !ok {
require.Fail(t, "no expected state transitions")
}
executeTest(t, r, tc.results, expectedTransitions, false)
})
executeTest(t, r, tc.results, expectedTransitions)
})
}
}
@@ -3287,60 +2676,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.AlertingErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Pending,
StateReason: eval.Error.String(),
Error: datasourceError,
Annotations: datasourceErrorAnnotations,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
},
ngmodels.OkErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
ngmodels.AlertingErrState: {
t2: {
{
@@ -3432,61 +2767,6 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.AlertingErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: datasourceError,
Annotations: datasourceErrorAnnotations,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
},
},
},
},
ngmodels.OkErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
ngmodels.AlertingErrState: {
t2: {
{
pkg/services/ngalert/state/manager_test.go
-412
View File
@@ -379,8 +379,6 @@ func TestProcessEvalResults(t *testing.T) {
return r
}
datasourceError := expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))
labels1 := data.Labels{
"instance_label": "test-1",
}
@@ -401,13 +399,6 @@ func TestProcessEvalResults(t *testing.T) {
"system + rule + no-data": mergeLabels(mergeLabels(noDataLabels, baseRule.Labels), systemLabels),
}
datasourceErrorAnnotations := data.Labels{
"annotation": "test",
"datasource_uid": "datasource_uid_1",
"ref_id": "A",
"Error": datasourceError.Error(),
}
// keep it separate to make code folding work correctly.
type testCase struct {
desc string
@@ -750,66 +741,6 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> pending when For is set but not exceeded, result is NoData and NoDataState is alerting",
alertRule: baseRuleWith(m.WithForNTimes(6), m.WithNoDataExecAs(models.Alerting)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
{
desc: "normal -> alerting when For is exceeded, result is NoData and NoDataState is alerting",
alertRule: baseRuleWith(m.WithForNTimes(3), m.WithNoDataExecAs(models.Alerting)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because nodata has no labels of regular result
},
t3: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)),
},
tn(5): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(tn(5), eval.NoData),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
LastSentAt: util.Pointer(tn(5)),
},
},
},
{
desc: "normal -> nodata when result is NoData and NoDataState is nodata",
alertRule: baseRule,
@@ -950,95 +881,6 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> normal (NoData, KeepLastState) -> alerting -> alerting (NoData, KeepLastState) - keeps last state when result is NoData and NoDataState is KeepLast",
alertRule: baseRuleWith(m.WithForNTimes(0), m.WithNoDataExecAs(models.KeepLast)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3.
},
},
},
{
desc: "normal -> pending -> pending (NoData, KeepLastState) -> alerting (NoData, KeepLastState) - keep last state respects For when result is NoData",
alertRule: baseRuleWith(m.WithForNTimes(2), m.WithNoDataExecAs(models.KeepLast)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
tn(4): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
LastSentAt: util.Pointer(tn(4)),
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is ok",
alertRule: baseRuleWith(m.WithNoDataExecAs(models.OK)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
},
expectedAnnotations: 0,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
{
desc: "normal -> pending when For is set but not exceeded, result is Error and ExecErrState is Alerting",
alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.AlertingErrState)),
@@ -1103,260 +945,6 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> error when result is Error and ExecErrState is Error",
alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.ErrorErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
CacheID: func() data.Fingerprint {
lbls := models.InstanceLabels(labels["system + rule + labels1"])
return lbls.Fingerprint()
}(),
Labels: mergeLabels(labels["system + rule + labels1"], data.Labels{
"datasource_uid": "datasource_uid_1",
"ref_id": "A",
}),
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: datasourceError,
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
LastSentAt: &t2,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "Error": "[sse.dataQueryError] failed to execute query [A]: this is an error"},
},
},
},
{
desc: "normal -> normal (Error, KeepLastState) -> alerting -> alerting (Error, KeepLastState) - keeps last state when result is Error and ExecErrState is KeepLast",
alertRule: baseRuleWith(m.WithForNTimes(0), m.WithErrorExecAs(models.KeepLastErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3.
Annotations: datasourceErrorAnnotations,
},
},
},
{
desc: "normal -> pending -> pending (Error, KeepLastState) -> alerting (Error, KeepLastState) - keep last state respects For when result is Error",
alertRule: baseRuleWith(m.WithForNTimes(2), m.WithErrorExecAs(models.KeepLastErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
tn(4): {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
LastSentAt: util.Pointer(tn(4)),
Annotations: datasourceErrorAnnotations,
},
},
},
{
desc: "normal -> normal when result is Error and ExecErrState is OK",
alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.OkErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
{
desc: "alerting -> normal when result is Error and ExecErrState is OK",
alertRule: baseRuleWith(m.WithForNTimes(6), m.WithErrorExecAs(models.OkErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
Annotations: datasourceErrorAnnotations,
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
{
desc: "normal -> alerting -> error when result is Error and ExecErrorState is Error",
alertRule: baseRuleWith(m.WithForNTimes(2)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created
},
tn(5): {
newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created
},
tn(6): {
newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO this is not how error result is created
},
},
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
LatestResult: newEvaluation(tn(6), eval.Error),
StartsAt: tn(4),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
LastSentAt: util.Pointer(tn(6)), // After 30s resend delay, last sent at is t6.
Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"},
},
},
},
{
desc: "normal -> alerting -> error -> alerting - it should clear the error",
alertRule: baseRuleWith(m.WithForNTimes(3)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(5): {
newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO fix it
},
tn(8): {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
},
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(tn(8), eval.Alerting),
StartsAt: tn(8),
EndsAt: tn(8).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(8),
LastSentAt: util.Pointer(tn(5)),
},
},
},
{
desc: "normal -> alerting -> error -> no data - it should clear the error",
alertRule: baseRuleWith(m.WithForNTimes(3)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(5): {
newResult(eval.WithState(eval.Error), eval.WithLabels(labels1)), // TODO FIX it
},
tn(6): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because it's not possible
},
},
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(tn(6), eval.NoData),
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
LastSentAt: util.Pointer(tn(5)),
},
},
},
{
desc: "template is correctly expanded",
alertRule: baseRuleWith(