diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 6709960a27c..f396e0565d8 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -1030,4 +1030,9 @@ export interface FeatureToggles { * Enables the alerting list view v2 preview toggle */ alertingListViewV2PreviewToggle?: boolean; + /** + * Use FiredAt for StartsAt when sending alerts to Alertmaanger + * @default false + */ + alertRuleUseFiredAtForStartsAt?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 820d571f435..d75409f7155 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1774,6 +1774,13 @@ var ( Stage: FeatureStagePrivatePreview, Owner: grafanaAlertingSquad, }, + { + Name: "alertRuleUseFiredAtForStartsAt", + Description: "Use FiredAt for StartsAt when sending alerts to Alertmaanger", + Stage: FeatureStageExperimental, + Owner: grafanaAlertingSquad, + Expression: "false", + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 491dd8c4ef9..8816160b388 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -232,3 +232,4 @@ logsPanelControls,preview,@grafana/observability-logs,false,false,true metricsFromProfiles,experimental,@grafana/observability-traces-and-profiling,false,false,true pluginsAutoUpdate,experimental,@grafana/plugins-platform-backend,false,false,false alertingListViewV2PreviewToggle,privatePreview,@grafana/alerting-squad,false,false,true +alertRuleUseFiredAtForStartsAt,experimental,@grafana/alerting-squad,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 301e473665a..ec9705ab011 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -938,4 +938,8 @@ const ( // FlagAlertingListViewV2PreviewToggle // Enables the alerting list view v2 preview toggle FlagAlertingListViewV2PreviewToggle = "alertingListViewV2PreviewToggle" + + // FlagAlertRuleUseFiredAtForStartsAt + // Use FiredAt for StartsAt when sending alerts to Alertmaanger + FlagAlertRuleUseFiredAtForStartsAt = "alertRuleUseFiredAtForStartsAt" ) diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 62ae788337d..57a7ab30714 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -71,6 +71,19 @@ "expression": "true" } }, + { + "metadata": { + "name": "alertRuleUseFiredAtForStartsAt", + "resourceVersion": "1744728881642", + "creationTimestamp": "2025-04-15T14:54:41Z" + }, + "spec": { + "description": "Use FiredAt for StartsAt when sending alerts to Alertmaanger", + "stage": "experimental", + "codeowner": "@grafana/alerting-squad", + "expression": "false" + } + }, { "metadata": { "name": "alertStateHistoryLokiOnly", diff --git a/pkg/services/ngalert/api/api_testing.go b/pkg/services/ngalert/api/api_testing.go index e8a6656beda..e26f2a89f9a 100644 --- a/pkg/services/ngalert/api/api_testing.go +++ b/pkg/services/ngalert/api/api_testing.go @@ -119,7 +119,7 @@ func (srv TestingApiSrv) RouteTestGrafanaRuleConfig(c *contextmodel.ReqContext, alerts := make([]*amv2.PostableAlert, 0, len(transitions)) for _, alertState := range transitions { - alerts = append(alerts, state.StateToPostableAlert(alertState, srv.appUrl)) + alerts = append(alerts, state.StateToPostableAlert(alertState, srv.appUrl, srv.featureManager)) } return response.JSON(http.StatusOK, alerts) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 69ef8ff5c90..5985904c4e9 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -396,6 +396,7 @@ func (ng *AlertNG) init() error { Tracer: ng.tracer, Log: log.New("ngalert.scheduler"), RecordingWriter: ng.RecordingWriter, + FeatureToggles: ng.FeatureToggles, } history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer, ac.NewRuleService(ng.accesscontrol)) diff --git a/pkg/services/ngalert/schedule/alert_rule.go b/pkg/services/ngalert/schedule/alert_rule.go index e8705eb0f64..925a4c36cfd 100644 --- a/pkg/services/ngalert/schedule/alert_rule.go +++ b/pkg/services/ngalert/schedule/alert_rule.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/metrics" @@ -64,6 +65,7 @@ func newRuleFactory( met *metrics.Scheduler, logger log.Logger, tracer tracing.Tracer, + featureToggles featuremgmt.FeatureToggles, recordingWriter RecordingWriter, evalAppliedHook evalAppliedFunc, stopAppliedHook stopAppliedFunc, @@ -98,6 +100,7 @@ func newRuleFactory( met, logger, tracer, + featureToggles, evalAppliedHook, stopAppliedHook, ) @@ -128,9 +131,10 @@ type alertRule struct { evalAppliedHook evalAppliedFunc stopAppliedHook stopAppliedFunc - metrics *metrics.Scheduler - logger log.Logger - tracer tracing.Tracer + metrics *metrics.Scheduler + logger log.Logger + tracer tracing.Tracer + featureToggles featuremgmt.FeatureToggles } func newAlertRule( @@ -146,6 +150,7 @@ func newAlertRule( met *metrics.Scheduler, logger log.Logger, tracer tracing.Tracer, + featureToggles featuremgmt.FeatureToggles, evalAppliedHook func(ngmodels.AlertRuleKey, time.Time), stopAppliedHook func(ngmodels.AlertRuleKey), ) *alertRule { @@ -168,6 +173,7 @@ func newAlertRule( metrics: met, logger: logger.FromContext(ctx), tracer: tracer, + featureToggles: featureToggles, } } @@ -471,7 +477,7 @@ func (a *alertRule) evaluate(ctx context.Context, e *Evaluation, span trace.Span func (a *alertRule) send(ctx context.Context, logger log.Logger, states state.StateTransitions) definitions.PostableAlerts { alerts := definitions.PostableAlerts{PostableAlerts: make([]models.PostableAlert, 0, len(states))} for _, alertState := range states { - alerts.PostableAlerts = append(alerts.PostableAlerts, *state.StateToPostableAlert(alertState, a.appURL)) + alerts.PostableAlerts = append(alerts.PostableAlerts, *state.StateToPostableAlert(alertState, a.appURL, a.featureToggles)) } if len(alerts.PostableAlerts) > 0 { @@ -483,7 +489,7 @@ func (a *alertRule) send(ctx context.Context, logger log.Logger, states state.St // sendExpire sends alerts to expire all previously firing alerts in the provided state transitions. func (a *alertRule) expireAndSend(ctx context.Context, states []state.StateTransition) { - expiredAlerts := state.FromAlertsStateToStoppedAlert(states, a.appURL, a.clock) + expiredAlerts := state.FromAlertsStateToStoppedAlert(states, a.appURL, a.clock, a.featureToggles) if len(expiredAlerts.PostableAlerts) > 0 { a.sender.Send(ctx, a.key.AlertRuleKey, expiredAlerts) } diff --git a/pkg/services/ngalert/schedule/alert_rule_test.go b/pkg/services/ngalert/schedule/alert_rule_test.go index ace81596042..c1cc0bbd4f0 100644 --- a/pkg/services/ngalert/schedule/alert_rule_test.go +++ b/pkg/services/ngalert/schedule/alert_rule_test.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/log/logtest" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -501,7 +502,7 @@ func blankRuleForTests(ctx context.Context, key models.AlertRuleKeyWithGroup) *a Log: log.NewNopLogger(), } st := state.NewManager(managerCfg, state.NewNoopPersister()) - return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, log.NewNopLogger(), nil, nil, nil) + return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, log.NewNopLogger(), nil, featuremgmt.WithFeatures(), nil, nil) } func TestRuleRoutine(t *testing.T) { @@ -1125,7 +1126,7 @@ func TestRuleRoutine(t *testing.T) { } func ruleFactoryFromScheduler(sch *schedule) ruleFactory { - return newRuleFactory(sch.appURL, sch.disableGrafanaFolder, sch.maxAttempts, sch.alertsSender, sch.stateManager, sch.evaluatorFactory, sch.clock, sch.rrCfg, sch.metrics, sch.log, sch.tracer, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc) + return newRuleFactory(sch.appURL, sch.disableGrafanaFolder, sch.maxAttempts, sch.alertsSender, sch.stateManager, sch.evaluatorFactory, sch.clock, sch.rrCfg, sch.metrics, sch.log, sch.tracer, sch.featureToggles, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc) } func stateForRule(rule *models.AlertRule, ts time.Time, evalState eval.State) *state.State { diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index 859d34a145c..67aba2ab917 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/metrics" @@ -106,8 +107,8 @@ type schedule struct { // last evaluated. schedulableAlertRules alertRulesRegistry - tracer tracing.Tracer - + tracer tracing.Tracer + featureToggles featuremgmt.FeatureToggles recordingWriter RecordingWriter } @@ -129,6 +130,7 @@ type SchedulerCfg struct { Log log.Logger RecordingWriter RecordingWriter RuleStopReasonProvider AlertRuleStopReasonProvider + FeatureToggles featuremgmt.FeatureToggles } // NewScheduler returns a new scheduler. @@ -159,6 +161,7 @@ func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule { tracer: cfg.Tracer, recordingWriter: cfg.RecordingWriter, ruleStopReasonProvider: cfg.RuleStopReasonProvider, + featureToggles: cfg.FeatureToggles, } return &sch @@ -305,6 +308,7 @@ func (sch *schedule) processTick(ctx context.Context, dispatcherGroup *errgroup. sch.metrics, sch.log, sch.tracer, + sch.featureToggles, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc, diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 3c1a1b7d4b2..2d9f1ad4af7 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -1216,6 +1216,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor AlertSender: senderMock, Tracer: testTracer, Log: log.New("ngalert.scheduler"), + FeatureToggles: featuremgmt.WithFeatures(), RecordingWriter: fakeRecordingWriter, RuleStopReasonProvider: ruleStopReasonProvider, } diff --git a/pkg/services/ngalert/state/compat.go b/pkg/services/ngalert/state/compat.go index 7f4b5c41428..60bd143ae4e 100644 --- a/pkg/services/ngalert/state/compat.go +++ b/pkg/services/ngalert/state/compat.go @@ -15,6 +15,7 @@ import ( alertingModels "github.com/grafana/alerting/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -33,7 +34,7 @@ const ( // - if evaluation state is either NoData or Error, the resulting set of labels is changed: // - original alert name (label: model.AlertNameLabel) is backed up to OriginalAlertName // - label model.AlertNameLabel is overwritten to either NoDataAlertName or ErrorAlertName -func StateToPostableAlert(transition StateTransition, appURL *url.URL) *models.PostableAlert { +func StateToPostableAlert(transition StateTransition, appURL *url.URL, featureToggles featuremgmt.FeatureToggles) *models.PostableAlert { alertState := transition.State nL := alertState.Labels.Copy() nA := data.Labels(alertState.Annotations).Copy() @@ -88,9 +89,14 @@ func StateToPostableAlert(transition StateTransition, appURL *url.URL) *models.P return errorAlert(nL, nA, alertState, urlStr) } + startsAt := strfmt.DateTime(alertState.StartsAt) + if featureToggles.IsEnabledGlobally(featuremgmt.FlagAlertRuleUseFiredAtForStartsAt) { + startsAt = strfmt.DateTime(alertState.FiredAt) + } + return &models.PostableAlert{ Annotations: models.LabelSet(nA), - StartsAt: strfmt.DateTime(alertState.StartsAt), + StartsAt: startsAt, EndsAt: strfmt.DateTime(alertState.EndsAt), Alert: models.Alert{ Labels: models.LabelSet(nL), @@ -141,14 +147,14 @@ func errorAlert(labels, annotations data.Labels, alertState *State, urlStr strin // FromAlertsStateToStoppedAlert selects only transitions from firing states (states eval.Alerting, eval.NoData, eval.Error) // and converts them to models.PostableAlert with EndsAt set to time.Now -func FromAlertsStateToStoppedAlert(firingStates []StateTransition, appURL *url.URL, clock clock.Clock) apimodels.PostableAlerts { +func FromAlertsStateToStoppedAlert(firingStates []StateTransition, appURL *url.URL, clock clock.Clock, featureToggles featuremgmt.FeatureToggles) apimodels.PostableAlerts { alerts := apimodels.PostableAlerts{PostableAlerts: make([]models.PostableAlert, 0, len(firingStates))} ts := clock.Now() for _, transition := range firingStates { if transition.PreviousState == eval.Normal || transition.PreviousState == eval.Pending { continue } - postableAlert := StateToPostableAlert(transition, appURL) + postableAlert := StateToPostableAlert(transition, appURL, featureToggles) postableAlert.EndsAt = strfmt.DateTime(ts) alerts.PostableAlerts = append(alerts.PostableAlerts, *postableAlert) } diff --git a/pkg/services/ngalert/state/compat_test.go b/pkg/services/ngalert/state/compat_test.go index 62f4f25edd1..7ea378b3e6d 100644 --- a/pkg/services/ngalert/state/compat_test.go +++ b/pkg/services/ngalert/state/compat_test.go @@ -16,6 +16,7 @@ import ( alertingModels "github.com/grafana/alerting/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/eval" ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/util" @@ -60,7 +61,7 @@ func Test_StateToPostableAlert(t *testing.T) { t.Run("to alert rule", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) u := *appURL u.Path = u.Path + "/alerting/grafana/" + alertState.AlertRuleUID + "/view" require.Equal(t, u.String(), result.GeneratorURL.String()) @@ -69,33 +70,49 @@ func Test_StateToPostableAlert(t *testing.T) { t.Run("app URL as is if rule UID is not specified", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.Labels[alertingModels.RuleUIDLabel] = "" - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, appURL.String(), result.GeneratorURL.String()) delete(alertState.Labels, alertingModels.RuleUIDLabel) - result = StateToPostableAlert(alertState, appURL) + result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, appURL.String(), result.GeneratorURL.String()) }) t.Run("empty string if app URL is not provided", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID - result := StateToPostableAlert(alertState, nil) + result := StateToPostableAlert(alertState, nil, featuremgmt.WithFeatures()) require.Equal(t, "", result.GeneratorURL.String()) }) }) t.Run("Start and End timestamps should be the same", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, strfmt.DateTime(alertState.StartsAt), result.StartsAt) require.Equal(t, strfmt.DateTime(alertState.EndsAt), result.EndsAt) }) + t.Run("StartsAt should be FiredAt if the feature flag is enabled", func(t *testing.T) { + if tc.state == eval.NoData || tc.state == eval.Error { + t.Skip("NoData and Error states are not supported for this test") + } + + alertState := randomTransition(eval.Normal, tc.state) + + // feature flag is disabled + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) + require.Equal(t, strfmt.DateTime(alertState.StartsAt), result.StartsAt) + + // feature flag is enabled + result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures(featuremgmt.FlagAlertRuleUseFiredAtForStartsAt)) + require.Equal(t, strfmt.DateTime(alertState.FiredAt), result.StartsAt) + }) + t.Run("should copy annotations", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.Annotations = randomMapOfStrings() - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, models.LabelSet(alertState.Annotations), result.Annotations) t.Run("add __value_string__ if it has results", func(t *testing.T) { @@ -104,7 +121,7 @@ func Test_StateToPostableAlert(t *testing.T) { expectedValueString := util.GenerateShortUID() alertState.LastEvaluationString = expectedValueString - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) expected := make(models.LabelSet, len(alertState.Annotations)+1) for k, v := range alertState.Annotations { @@ -116,7 +133,7 @@ func Test_StateToPostableAlert(t *testing.T) { // even overwrites alertState.Annotations["__value_string__"] = util.GenerateShortUID() - result = StateToPostableAlert(alertState, appURL) + result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, expected, result.Annotations) }) @@ -125,7 +142,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertState.Annotations = randomMapOfStrings() alertState.Image = &ngModels.Image{Token: "test_token", URL: "test_url"} - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) expected := make(models.LabelSet, len(alertState.Annotations)+1) for k, v := range alertState.Annotations { @@ -146,7 +163,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertState.Annotations = randomMapOfStrings() alertState.Image = &ngModels.Image{} - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) expected := make(models.LabelSet, len(alertState.Annotations)+1) for k, v := range alertState.Annotations { @@ -160,7 +177,7 @@ func Test_StateToPostableAlert(t *testing.T) { t.Run("should add state reason annotation if not empty", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.StateReason = "TEST_STATE_REASON" - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, alertState.StateReason, result.Annotations[ngModels.StateReasonAnnotation]) }) @@ -172,7 +189,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertName := util.GenerateShortUID() alertState.Labels[model.AlertNameLabel] = alertName - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) expected := make(models.LabelSet, len(alertState.Labels)+1) for k, v := range alertState.Labels { @@ -188,7 +205,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertState.Labels = randomMapOfStrings() delete(alertState.Labels, model.AlertNameLabel) - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, NoDataAlertName, result.Labels[model.AlertNameLabel]) require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename) @@ -201,7 +218,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertName := util.GenerateShortUID() alertState.Labels[model.AlertNameLabel] = alertName - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) expected := make(models.LabelSet, len(alertState.Labels)+1) for k, v := range alertState.Labels { @@ -217,7 +234,7 @@ func Test_StateToPostableAlert(t *testing.T) { alertState.Labels = randomMapOfStrings() delete(alertState.Labels, model.AlertNameLabel) - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, ErrorAlertName, result.Labels[model.AlertNameLabel]) require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename) @@ -227,7 +244,7 @@ func Test_StateToPostableAlert(t *testing.T) { t.Run("should copy labels as is", func(t *testing.T) { alertState := randomTransition(eval.Normal, tc.state) alertState.Labels = randomMapOfStrings() - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, models.LabelSet(alertState.Labels), result.Labels) }) } @@ -277,7 +294,7 @@ func TestStateToPostableAlertFromNodataError(t *testing.T) { alertState.ResolvedAt = &alertState.LastEvaluationTime } alertState.Labels = data.Labels(standardLabels) - result := StateToPostableAlert(alertState, appURL) + result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures()) require.Equal(t, tc.expectedLabels, result.Labels) }) } @@ -306,12 +323,12 @@ func Test_FromAlertsStateToStoppedAlert(t *testing.T) { if s.PreviousState != eval.Alerting && s.PreviousState != eval.Error && s.PreviousState != eval.NoData { continue } - alert := StateToPostableAlert(s, appURL) + alert := StateToPostableAlert(s, appURL, featuremgmt.WithFeatures()) alert.EndsAt = strfmt.DateTime(clk.Now()) expected = append(expected, *alert) } - result := FromAlertsStateToStoppedAlert(states, appURL, clk) + result := FromAlertsStateToStoppedAlert(states, appURL, clk, featuremgmt.WithFeatures()) require.Equal(t, expected, result.PostableAlerts) } @@ -344,6 +361,7 @@ func randomTransition(from, to eval.State) StateTransition { State: to, AlertRuleUID: util.GenerateShortUID(), StartsAt: time.Now(), + FiredAt: randomTimeInPast(), EndsAt: randomTimeInFuture(), LastEvaluationTime: randomTimeInPast(), EvaluationDuration: randomDuration(), diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 7c41ee4a859..7b0608e9596 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -428,6 +428,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -505,6 +506,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Alerting), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -573,6 +575,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -627,6 +630,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -687,6 +691,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -748,6 +753,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -806,6 +812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -820,6 +827,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -834,6 +842,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -848,6 +857,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.Normal), StartsAt: t4, EndsAt: t4, + FiredAt: t1, LastEvaluationTime: t4, LastSentAt: &t4, ResolvedAt: &t4, @@ -880,6 +890,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -894,6 +905,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -908,6 +920,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -942,6 +955,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Alerting), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -956,6 +970,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: &t2, }, @@ -970,6 +985,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.Alerting), StartsAt: t4, EndsAt: t4.Add(ResendDelay * 4), + FiredAt: t4, LastEvaluationTime: t4, LastSentAt: &t2, }, @@ -1001,6 +1017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -1044,6 +1061,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -1108,6 +1126,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t4, + FiredAt: t1, LastEvaluationTime: t4, ResolvedAt: &t4, LastSentAt: &t4, @@ -1140,6 +1159,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -1169,6 +1189,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -1216,6 +1237,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -1228,6 +1250,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -1242,6 +1265,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Alerting), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -1257,6 +1281,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -1272,6 +1297,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -1300,6 +1326,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Alerting), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -1383,6 +1410,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.NoData), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -1467,6 +1495,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, Values: map[string]float64{"A": float64(-1)}, @@ -1564,6 +1593,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -1595,6 +1625,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -1609,6 +1640,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -1626,6 +1658,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: &t2, }, @@ -1641,6 +1674,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -1672,6 +1706,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -1704,6 +1739,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t2, LastSentAt: &t2, @@ -1736,6 +1772,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -1767,6 +1804,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -1870,6 +1908,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -1887,6 +1926,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -1902,6 +1942,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: &t2, }, @@ -1993,6 +2034,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -2024,6 +2066,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: &t2, }, @@ -2057,6 +2100,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -2074,6 +2118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -2107,6 +2152,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -2159,6 +2205,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -2206,6 +2253,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -2223,6 +2271,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -2240,6 +2289,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.NoData), StartsAt: t1, EndsAt: t4.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t4, LastSentAt: &t4, }, @@ -2258,6 +2308,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -2275,6 +2326,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -2292,6 +2344,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.NoData), StartsAt: t4, EndsAt: t4, + FiredAt: t1, LastEvaluationTime: t4, LastSentAt: &t4, ResolvedAt: &t4, @@ -2311,6 +2364,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -2328,6 +2382,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -2345,6 +2400,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.NoData), StartsAt: t1, EndsAt: t4.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t4, LastSentAt: &t4, }, @@ -2458,6 +2514,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -2473,6 +2530,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.Normal), StartsAt: t4, EndsAt: t4.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t4, LastSentAt: &t3, }, @@ -2488,6 +2546,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t5, eval.NoData), StartsAt: t5, EndsAt: t5.Add(ResendDelay * 4), + FiredAt: t5, LastEvaluationTime: t5, LastSentAt: &t3, Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations), @@ -2657,6 +2716,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.NoData), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -2761,6 +2821,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -2842,6 +2903,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -2873,6 +2935,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -2890,6 +2953,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -2908,6 +2972,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -2926,6 +2991,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t2, LastSentAt: &t2, @@ -2945,6 +3011,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.NoData), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -2962,6 +3029,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.NoData), StartsAt: t1, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -3009,6 +3077,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -3041,6 +3110,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -3104,6 +3174,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -3196,6 +3267,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -3277,6 +3349,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -3365,6 +3438,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, Values: map[string]float64{"A": float64(-1)}, @@ -3402,6 +3476,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, Values: map[string]float64{"A": float64(-1)}, @@ -3456,6 +3531,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, Values: map[string]float64{"A": float64(-1)}, @@ -3574,6 +3650,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t3, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t3, LastSentAt: &t3, @@ -3714,6 +3791,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, @@ -3728,6 +3806,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t2, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t3, LastSentAt: &t1, }, @@ -3742,6 +3821,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.Normal), StartsAt: t4, EndsAt: t4, + FiredAt: t1, LastEvaluationTime: t4, ResolvedAt: &t4, LastSentAt: &t4, @@ -3920,6 +4000,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, Annotations: datasourceErrorAnnotations, @@ -3936,6 +4017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: &t2, }, @@ -3952,6 +4034,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t4, eval.Error), StartsAt: t4, EndsAt: t4.Add(ResendDelay * 4), + FiredAt: t4, LastEvaluationTime: t4, LastSentAt: &t2, Annotations: datasourceErrorAnnotations, @@ -4100,6 +4183,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -4201,6 +4285,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -4236,6 +4321,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Error), StartsAt: t2, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -4321,6 +4407,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -4384,6 +4471,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t3, }, @@ -4430,6 +4518,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -4534,6 +4623,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t1, eval.Error), StartsAt: t1, EndsAt: t1.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -4552,6 +4642,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { LatestResult: newEvaluation(t2, eval.Error), StartsAt: t1, EndsAt: t2.Add(ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t2, LastSentAt: &t1, }, diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 29f6cab577c..4ce84497c6d 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -350,6 +350,7 @@ func TestProcessEvalResults(t *testing.T) { t2 := tn(2) t3 := tn(3) + m := models.RuleMuts baseRule := &models.AlertRule{ OrgID: 1, @@ -475,6 +476,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, EndsAt: t1.Add(state.ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, }, @@ -523,6 +525,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t2, eval.Alerting), StartsAt: t2, EndsAt: t2.Add(state.ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t2, LastSentAt: &t2, }, @@ -554,6 +557,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(4), eval.Alerting), StartsAt: tn(4), EndsAt: tn(4).Add(state.ResendDelay * 4), + FiredAt: tn(4), LastEvaluationTime: tn(4), LastSentAt: util.Pointer(tn(4)), }, @@ -579,6 +583,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t2, eval.Normal), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t2, ResolvedAt: &t2, LastSentAt: &t2, @@ -608,6 +613,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t2, EndsAt: t2, + FiredAt: t1, LastEvaluationTime: t3, ResolvedAt: &t2, LastSentAt: &t2, @@ -640,6 +646,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(4), eval.Alerting), StartsAt: tn(4), EndsAt: tn(4).Add(state.ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: tn(4), ResolvedAt: &t3, LastSentAt: &t3, @@ -706,6 +713,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(4), eval.NoData), StartsAt: tn(4), EndsAt: tn(4).Add(state.ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: tn(4), LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3. }, @@ -959,6 +967,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(5), eval.Error), StartsAt: tn(5), EndsAt: tn(5).Add(state.ResendDelay * 4), + FiredAt: tn(5), LastEvaluationTime: tn(5), LastSentAt: util.Pointer(tn(5)), }, @@ -1026,6 +1035,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t3, eval.Alerting), StartsAt: t3, EndsAt: t3.Add(state.ResendDelay * 4), + FiredAt: t3, LastEvaluationTime: t3, LastSentAt: &t1, // Resend delay is 30s, so last sent at is t1. }, @@ -1057,6 +1067,7 @@ func TestProcessEvalResults(t *testing.T) { }), StartsAt: t1, EndsAt: t1.Add(state.ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, Values: map[string]float64{ @@ -1092,6 +1103,7 @@ func TestProcessEvalResults(t *testing.T) { }), StartsAt: t1, EndsAt: t1.Add(state.ResendDelay * 4), + FiredAt: t1, LastEvaluationTime: t1, LastSentAt: &t1, Values: map[string]float64{ @@ -1124,6 +1136,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(t3, eval.Normal), StartsAt: t3, EndsAt: t3.Add(state.ResendDelay * 4), + FiredAt: t2, LastEvaluationTime: t3, LastSentAt: util.Pointer(t2), }, @@ -1158,6 +1171,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(5), eval.Normal), StartsAt: tn(5), EndsAt: tn(5), + FiredAt: t2, LastEvaluationTime: tn(5), LastSentAt: util.Pointer(tn(5)), ResolvedAt: util.Pointer(tn(5)), @@ -1196,6 +1210,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(6), eval.Normal), StartsAt: tn(6), EndsAt: tn(6).Add(state.ResendDelay * 4), + FiredAt: tn(4), LastEvaluationTime: tn(6), LastSentAt: util.Pointer(tn(5)), }, @@ -1230,6 +1245,7 @@ func TestProcessEvalResults(t *testing.T) { LatestResult: newEvaluation(tn(5), eval.Alerting), StartsAt: tn(5), EndsAt: tn(5).Add(state.ResendDelay * 4), + FiredAt: tn(5), LastEvaluationTime: tn(5), LastSentAt: util.Pointer(t3), }, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 47957428c19..05437181100 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -63,6 +63,9 @@ type State struct { // conditions. Values map[string]float64 + // FiredAt is the time the state first transitions to Alerting. + FiredAt time.Time + StartsAt time.Time // EndsAt is different from the Prometheus EndsAt as EndsAt is updated for both Normal states // and states that have been resolved. It cannot be used to determine when a state was resolved. @@ -128,6 +131,7 @@ func (a *State) Copy() *State { Values: a.Values, StartsAt: a.StartsAt, EndsAt: a.EndsAt, + FiredAt: a.FiredAt, ResolvedAt: a.ResolvedAt, LastSentAt: a.LastSentAt, LastEvaluationString: a.LastEvaluationString, @@ -159,6 +163,9 @@ func (a *State) SetAlerting(reason string, startsAt, endsAt time.Time) { a.StartsAt = startsAt a.EndsAt = endsAt a.Error = nil + + // FiredAt is only ever set when the state is set to Alerting. + a.FiredAt = startsAt } // SetPending sets the state to Pending. It changes both the start and end time. @@ -772,6 +779,7 @@ func patch(newState, existingState *State, result eval.Result) { newState.LastEvaluationString = existingState.LastEvaluationString newState.StartsAt = existingState.StartsAt newState.EndsAt = existingState.EndsAt + newState.FiredAt = existingState.FiredAt newState.ResolvedAt = existingState.ResolvedAt newState.LastSentAt = existingState.LastSentAt // Annotations can change over time, however we also want to maintain diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index 8200696d385..5c2b043815b 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -46,6 +46,7 @@ func TestSetAlerting(t *testing.T) { StateReason: "this is a reason", StartsAt: mock.Now(), EndsAt: mock.Now().Add(time.Minute), + FiredAt: mock.Now(), }, }, { name: "previous state is removed", @@ -60,6 +61,7 @@ func TestSetAlerting(t *testing.T) { State: eval.Alerting, StartsAt: mock.Now(), EndsAt: mock.Now().Add(time.Minute), + FiredAt: mock.Now(), }, }}