Alerting: Add FiredAt field to State (#104046)

* Add FiredAt field to the State

* Update featuretoggle files

* Fix lint errors

* Fix test compilation

* Remove random print line + formatting

* Address PR comments
This commit is contained in:
Fayzal Ghantiwala
2025-04-22 12:16:38 +01:00
committed by GitHub
parent 9ed3c9ee8a
commit 3a054d5e00
17 changed files with 217 additions and 33 deletions
@@ -1030,4 +1030,9 @@ export interface FeatureToggles {
* Enables the alerting list view v2 preview toggle
*/
alertingListViewV2PreviewToggle?: boolean;
/**
* Use FiredAt for StartsAt when sending alerts to Alertmaanger
* @default false
*/
alertRuleUseFiredAtForStartsAt?: boolean;
}
+7
View File
@@ -1774,6 +1774,13 @@ var (
Stage: FeatureStagePrivatePreview,
Owner: grafanaAlertingSquad,
},
{
Name: "alertRuleUseFiredAtForStartsAt",
Description: "Use FiredAt for StartsAt when sending alerts to Alertmaanger",
Stage: FeatureStageExperimental,
Owner: grafanaAlertingSquad,
Expression: "false",
},
}
)
+1
View File
@@ -232,3 +232,4 @@ logsPanelControls,preview,@grafana/observability-logs,false,false,true
metricsFromProfiles,experimental,@grafana/observability-traces-and-profiling,false,false,true
pluginsAutoUpdate,experimental,@grafana/plugins-platform-backend,false,false,false
alertingListViewV2PreviewToggle,privatePreview,@grafana/alerting-squad,false,false,true
alertRuleUseFiredAtForStartsAt,experimental,@grafana/alerting-squad,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
232 metricsFromProfiles experimental @grafana/observability-traces-and-profiling false false true
233 pluginsAutoUpdate experimental @grafana/plugins-platform-backend false false false
234 alertingListViewV2PreviewToggle privatePreview @grafana/alerting-squad false false true
235 alertRuleUseFiredAtForStartsAt experimental @grafana/alerting-squad false false false
+4
View File
@@ -938,4 +938,8 @@ const (
// FlagAlertingListViewV2PreviewToggle
// Enables the alerting list view v2 preview toggle
FlagAlertingListViewV2PreviewToggle = "alertingListViewV2PreviewToggle"
// FlagAlertRuleUseFiredAtForStartsAt
// Use FiredAt for StartsAt when sending alerts to Alertmaanger
FlagAlertRuleUseFiredAtForStartsAt = "alertRuleUseFiredAtForStartsAt"
)
+13
View File
@@ -71,6 +71,19 @@
"expression": "true"
}
},
{
"metadata": {
"name": "alertRuleUseFiredAtForStartsAt",
"resourceVersion": "1744728881642",
"creationTimestamp": "2025-04-15T14:54:41Z"
},
"spec": {
"description": "Use FiredAt for StartsAt when sending alerts to Alertmaanger",
"stage": "experimental",
"codeowner": "@grafana/alerting-squad",
"expression": "false"
}
},
{
"metadata": {
"name": "alertStateHistoryLokiOnly",
+1 -1
View File
@@ -119,7 +119,7 @@ func (srv TestingApiSrv) RouteTestGrafanaRuleConfig(c *contextmodel.ReqContext,
alerts := make([]*amv2.PostableAlert, 0, len(transitions))
for _, alertState := range transitions {
alerts = append(alerts, state.StateToPostableAlert(alertState, srv.appUrl))
alerts = append(alerts, state.StateToPostableAlert(alertState, srv.appUrl, srv.featureManager))
}
return response.JSON(http.StatusOK, alerts)
+1
View File
@@ -396,6 +396,7 @@ func (ng *AlertNG) init() error {
Tracer: ng.tracer,
Log: log.New("ngalert.scheduler"),
RecordingWriter: ng.RecordingWriter,
FeatureToggles: ng.FeatureToggles,
}
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer, ac.NewRuleService(ng.accesscontrol))
+11 -5
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -64,6 +65,7 @@ func newRuleFactory(
met *metrics.Scheduler,
logger log.Logger,
tracer tracing.Tracer,
featureToggles featuremgmt.FeatureToggles,
recordingWriter RecordingWriter,
evalAppliedHook evalAppliedFunc,
stopAppliedHook stopAppliedFunc,
@@ -98,6 +100,7 @@ func newRuleFactory(
met,
logger,
tracer,
featureToggles,
evalAppliedHook,
stopAppliedHook,
)
@@ -128,9 +131,10 @@ type alertRule struct {
evalAppliedHook evalAppliedFunc
stopAppliedHook stopAppliedFunc
metrics *metrics.Scheduler
logger log.Logger
tracer tracing.Tracer
metrics *metrics.Scheduler
logger log.Logger
tracer tracing.Tracer
featureToggles featuremgmt.FeatureToggles
}
func newAlertRule(
@@ -146,6 +150,7 @@ func newAlertRule(
met *metrics.Scheduler,
logger log.Logger,
tracer tracing.Tracer,
featureToggles featuremgmt.FeatureToggles,
evalAppliedHook func(ngmodels.AlertRuleKey, time.Time),
stopAppliedHook func(ngmodels.AlertRuleKey),
) *alertRule {
@@ -168,6 +173,7 @@ func newAlertRule(
metrics: met,
logger: logger.FromContext(ctx),
tracer: tracer,
featureToggles: featureToggles,
}
}
@@ -471,7 +477,7 @@ func (a *alertRule) evaluate(ctx context.Context, e *Evaluation, span trace.Span
func (a *alertRule) send(ctx context.Context, logger log.Logger, states state.StateTransitions) definitions.PostableAlerts {
alerts := definitions.PostableAlerts{PostableAlerts: make([]models.PostableAlert, 0, len(states))}
for _, alertState := range states {
alerts.PostableAlerts = append(alerts.PostableAlerts, *state.StateToPostableAlert(alertState, a.appURL))
alerts.PostableAlerts = append(alerts.PostableAlerts, *state.StateToPostableAlert(alertState, a.appURL, a.featureToggles))
}
if len(alerts.PostableAlerts) > 0 {
@@ -483,7 +489,7 @@ func (a *alertRule) send(ctx context.Context, logger log.Logger, states state.St
// sendExpire sends alerts to expire all previously firing alerts in the provided state transitions.
func (a *alertRule) expireAndSend(ctx context.Context, states []state.StateTransition) {
expiredAlerts := state.FromAlertsStateToStoppedAlert(states, a.appURL, a.clock)
expiredAlerts := state.FromAlertsStateToStoppedAlert(states, a.appURL, a.clock, a.featureToggles)
if len(expiredAlerts.PostableAlerts) > 0 {
a.sender.Send(ctx, a.key.AlertRuleKey, expiredAlerts)
}
@@ -25,6 +25,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/log/logtest"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -501,7 +502,7 @@ func blankRuleForTests(ctx context.Context, key models.AlertRuleKeyWithGroup) *a
Log: log.NewNopLogger(),
}
st := state.NewManager(managerCfg, state.NewNoopPersister())
return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, log.NewNopLogger(), nil, nil, nil)
return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, log.NewNopLogger(), nil, featuremgmt.WithFeatures(), nil, nil)
}
func TestRuleRoutine(t *testing.T) {
@@ -1125,7 +1126,7 @@ func TestRuleRoutine(t *testing.T) {
}
func ruleFactoryFromScheduler(sch *schedule) ruleFactory {
return newRuleFactory(sch.appURL, sch.disableGrafanaFolder, sch.maxAttempts, sch.alertsSender, sch.stateManager, sch.evaluatorFactory, sch.clock, sch.rrCfg, sch.metrics, sch.log, sch.tracer, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc)
return newRuleFactory(sch.appURL, sch.disableGrafanaFolder, sch.maxAttempts, sch.alertsSender, sch.stateManager, sch.evaluatorFactory, sch.clock, sch.rrCfg, sch.metrics, sch.log, sch.tracer, sch.featureToggles, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc)
}
func stateForRule(rule *models.AlertRule, ts time.Time, evalState eval.State) *state.State {
+6 -2
View File
@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -106,8 +107,8 @@ type schedule struct {
// last evaluated.
schedulableAlertRules alertRulesRegistry
tracer tracing.Tracer
tracer tracing.Tracer
featureToggles featuremgmt.FeatureToggles
recordingWriter RecordingWriter
}
@@ -129,6 +130,7 @@ type SchedulerCfg struct {
Log log.Logger
RecordingWriter RecordingWriter
RuleStopReasonProvider AlertRuleStopReasonProvider
FeatureToggles featuremgmt.FeatureToggles
}
// NewScheduler returns a new scheduler.
@@ -159,6 +161,7 @@ func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule {
tracer: cfg.Tracer,
recordingWriter: cfg.RecordingWriter,
ruleStopReasonProvider: cfg.RuleStopReasonProvider,
featureToggles: cfg.FeatureToggles,
}
return &sch
@@ -305,6 +308,7 @@ func (sch *schedule) processTick(ctx context.Context, dispatcherGroup *errgroup.
sch.metrics,
sch.log,
sch.tracer,
sch.featureToggles,
sch.recordingWriter,
sch.evalAppliedFunc,
sch.stopAppliedFunc,
@@ -1216,6 +1216,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
AlertSender: senderMock,
Tracer: testTracer,
Log: log.New("ngalert.scheduler"),
FeatureToggles: featuremgmt.WithFeatures(),
RecordingWriter: fakeRecordingWriter,
RuleStopReasonProvider: ruleStopReasonProvider,
}
+10 -4
View File
@@ -15,6 +15,7 @@ import (
alertingModels "github.com/grafana/alerting/models"
"github.com/grafana/grafana/pkg/services/featuremgmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngModels "github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -33,7 +34,7 @@ const (
// - if evaluation state is either NoData or Error, the resulting set of labels is changed:
// - original alert name (label: model.AlertNameLabel) is backed up to OriginalAlertName
// - label model.AlertNameLabel is overwritten to either NoDataAlertName or ErrorAlertName
func StateToPostableAlert(transition StateTransition, appURL *url.URL) *models.PostableAlert {
func StateToPostableAlert(transition StateTransition, appURL *url.URL, featureToggles featuremgmt.FeatureToggles) *models.PostableAlert {
alertState := transition.State
nL := alertState.Labels.Copy()
nA := data.Labels(alertState.Annotations).Copy()
@@ -88,9 +89,14 @@ func StateToPostableAlert(transition StateTransition, appURL *url.URL) *models.P
return errorAlert(nL, nA, alertState, urlStr)
}
startsAt := strfmt.DateTime(alertState.StartsAt)
if featureToggles.IsEnabledGlobally(featuremgmt.FlagAlertRuleUseFiredAtForStartsAt) {
startsAt = strfmt.DateTime(alertState.FiredAt)
}
return &models.PostableAlert{
Annotations: models.LabelSet(nA),
StartsAt: strfmt.DateTime(alertState.StartsAt),
StartsAt: startsAt,
EndsAt: strfmt.DateTime(alertState.EndsAt),
Alert: models.Alert{
Labels: models.LabelSet(nL),
@@ -141,14 +147,14 @@ func errorAlert(labels, annotations data.Labels, alertState *State, urlStr strin
// FromAlertsStateToStoppedAlert selects only transitions from firing states (states eval.Alerting, eval.NoData, eval.Error)
// and converts them to models.PostableAlert with EndsAt set to time.Now
func FromAlertsStateToStoppedAlert(firingStates []StateTransition, appURL *url.URL, clock clock.Clock) apimodels.PostableAlerts {
func FromAlertsStateToStoppedAlert(firingStates []StateTransition, appURL *url.URL, clock clock.Clock, featureToggles featuremgmt.FeatureToggles) apimodels.PostableAlerts {
alerts := apimodels.PostableAlerts{PostableAlerts: make([]models.PostableAlert, 0, len(firingStates))}
ts := clock.Now()
for _, transition := range firingStates {
if transition.PreviousState == eval.Normal || transition.PreviousState == eval.Pending {
continue
}
postableAlert := StateToPostableAlert(transition, appURL)
postableAlert := StateToPostableAlert(transition, appURL, featureToggles)
postableAlert.EndsAt = strfmt.DateTime(ts)
alerts.PostableAlerts = append(alerts.PostableAlerts, *postableAlert)
}
+37 -19
View File
@@ -16,6 +16,7 @@ import (
alertingModels "github.com/grafana/alerting/models"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngModels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/util"
@@ -60,7 +61,7 @@ func Test_StateToPostableAlert(t *testing.T) {
t.Run("to alert rule", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
u := *appURL
u.Path = u.Path + "/alerting/grafana/" + alertState.AlertRuleUID + "/view"
require.Equal(t, u.String(), result.GeneratorURL.String())
@@ -69,33 +70,49 @@ func Test_StateToPostableAlert(t *testing.T) {
t.Run("app URL as is if rule UID is not specified", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.Labels[alertingModels.RuleUIDLabel] = ""
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, appURL.String(), result.GeneratorURL.String())
delete(alertState.Labels, alertingModels.RuleUIDLabel)
result = StateToPostableAlert(alertState, appURL)
result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, appURL.String(), result.GeneratorURL.String())
})
t.Run("empty string if app URL is not provided", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID
result := StateToPostableAlert(alertState, nil)
result := StateToPostableAlert(alertState, nil, featuremgmt.WithFeatures())
require.Equal(t, "", result.GeneratorURL.String())
})
})
t.Run("Start and End timestamps should be the same", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, strfmt.DateTime(alertState.StartsAt), result.StartsAt)
require.Equal(t, strfmt.DateTime(alertState.EndsAt), result.EndsAt)
})
t.Run("StartsAt should be FiredAt if the feature flag is enabled", func(t *testing.T) {
if tc.state == eval.NoData || tc.state == eval.Error {
t.Skip("NoData and Error states are not supported for this test")
}
alertState := randomTransition(eval.Normal, tc.state)
// feature flag is disabled
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, strfmt.DateTime(alertState.StartsAt), result.StartsAt)
// feature flag is enabled
result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures(featuremgmt.FlagAlertRuleUseFiredAtForStartsAt))
require.Equal(t, strfmt.DateTime(alertState.FiredAt), result.StartsAt)
})
t.Run("should copy annotations", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.Annotations = randomMapOfStrings()
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, models.LabelSet(alertState.Annotations), result.Annotations)
t.Run("add __value_string__ if it has results", func(t *testing.T) {
@@ -104,7 +121,7 @@ func Test_StateToPostableAlert(t *testing.T) {
expectedValueString := util.GenerateShortUID()
alertState.LastEvaluationString = expectedValueString
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
expected := make(models.LabelSet, len(alertState.Annotations)+1)
for k, v := range alertState.Annotations {
@@ -116,7 +133,7 @@ func Test_StateToPostableAlert(t *testing.T) {
// even overwrites
alertState.Annotations["__value_string__"] = util.GenerateShortUID()
result = StateToPostableAlert(alertState, appURL)
result = StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, expected, result.Annotations)
})
@@ -125,7 +142,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertState.Annotations = randomMapOfStrings()
alertState.Image = &ngModels.Image{Token: "test_token", URL: "test_url"}
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
expected := make(models.LabelSet, len(alertState.Annotations)+1)
for k, v := range alertState.Annotations {
@@ -146,7 +163,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertState.Annotations = randomMapOfStrings()
alertState.Image = &ngModels.Image{}
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
expected := make(models.LabelSet, len(alertState.Annotations)+1)
for k, v := range alertState.Annotations {
@@ -160,7 +177,7 @@ func Test_StateToPostableAlert(t *testing.T) {
t.Run("should add state reason annotation if not empty", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.StateReason = "TEST_STATE_REASON"
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, alertState.StateReason, result.Annotations[ngModels.StateReasonAnnotation])
})
@@ -172,7 +189,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertName := util.GenerateShortUID()
alertState.Labels[model.AlertNameLabel] = alertName
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
expected := make(models.LabelSet, len(alertState.Labels)+1)
for k, v := range alertState.Labels {
@@ -188,7 +205,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertState.Labels = randomMapOfStrings()
delete(alertState.Labels, model.AlertNameLabel)
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, NoDataAlertName, result.Labels[model.AlertNameLabel])
require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename)
@@ -201,7 +218,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertName := util.GenerateShortUID()
alertState.Labels[model.AlertNameLabel] = alertName
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
expected := make(models.LabelSet, len(alertState.Labels)+1)
for k, v := range alertState.Labels {
@@ -217,7 +234,7 @@ func Test_StateToPostableAlert(t *testing.T) {
alertState.Labels = randomMapOfStrings()
delete(alertState.Labels, model.AlertNameLabel)
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, ErrorAlertName, result.Labels[model.AlertNameLabel])
require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename)
@@ -227,7 +244,7 @@ func Test_StateToPostableAlert(t *testing.T) {
t.Run("should copy labels as is", func(t *testing.T) {
alertState := randomTransition(eval.Normal, tc.state)
alertState.Labels = randomMapOfStrings()
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, models.LabelSet(alertState.Labels), result.Labels)
})
}
@@ -277,7 +294,7 @@ func TestStateToPostableAlertFromNodataError(t *testing.T) {
alertState.ResolvedAt = &alertState.LastEvaluationTime
}
alertState.Labels = data.Labels(standardLabels)
result := StateToPostableAlert(alertState, appURL)
result := StateToPostableAlert(alertState, appURL, featuremgmt.WithFeatures())
require.Equal(t, tc.expectedLabels, result.Labels)
})
}
@@ -306,12 +323,12 @@ func Test_FromAlertsStateToStoppedAlert(t *testing.T) {
if s.PreviousState != eval.Alerting && s.PreviousState != eval.Error && s.PreviousState != eval.NoData {
continue
}
alert := StateToPostableAlert(s, appURL)
alert := StateToPostableAlert(s, appURL, featuremgmt.WithFeatures())
alert.EndsAt = strfmt.DateTime(clk.Now())
expected = append(expected, *alert)
}
result := FromAlertsStateToStoppedAlert(states, appURL, clk)
result := FromAlertsStateToStoppedAlert(states, appURL, clk, featuremgmt.WithFeatures())
require.Equal(t, expected, result.PostableAlerts)
}
@@ -344,6 +361,7 @@ func randomTransition(from, to eval.State) StateTransition {
State: to,
AlertRuleUID: util.GenerateShortUID(),
StartsAt: time.Now(),
FiredAt: randomTimeInPast(),
EndsAt: randomTimeInFuture(),
LastEvaluationTime: randomTimeInPast(),
EvaluationDuration: randomDuration(),
@@ -428,6 +428,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -505,6 +506,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -573,6 +575,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -627,6 +630,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -687,6 +691,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -748,6 +753,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -806,6 +812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -820,6 +827,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -834,6 +842,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -848,6 +857,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t4,
EndsAt: t4,
FiredAt: t1,
LastEvaluationTime: t4,
LastSentAt: &t4,
ResolvedAt: &t4,
@@ -880,6 +890,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -894,6 +905,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -908,6 +920,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -942,6 +955,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -956,6 +970,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t2,
},
@@ -970,6 +985,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.Alerting),
StartsAt: t4,
EndsAt: t4.Add(ResendDelay * 4),
FiredAt: t4,
LastEvaluationTime: t4,
LastSentAt: &t2,
},
@@ -1001,6 +1017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -1044,6 +1061,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -1108,6 +1126,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t4,
FiredAt: t1,
LastEvaluationTime: t4,
ResolvedAt: &t4,
LastSentAt: &t4,
@@ -1140,6 +1159,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -1169,6 +1189,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -1216,6 +1237,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -1228,6 +1250,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -1242,6 +1265,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -1257,6 +1281,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -1272,6 +1297,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -1300,6 +1326,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -1383,6 +1410,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -1467,6 +1495,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
Values: map[string]float64{"A": float64(-1)},
@@ -1564,6 +1593,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -1595,6 +1625,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -1609,6 +1640,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -1626,6 +1658,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t2,
},
@@ -1641,6 +1674,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -1672,6 +1706,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -1704,6 +1739,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -1736,6 +1772,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -1767,6 +1804,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -1870,6 +1908,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -1887,6 +1926,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -1902,6 +1942,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t2,
},
@@ -1993,6 +2034,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -2024,6 +2066,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t2,
},
@@ -2057,6 +2100,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -2074,6 +2118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -2107,6 +2152,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -2159,6 +2205,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -2206,6 +2253,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -2223,6 +2271,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -2240,6 +2289,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t1,
EndsAt: t4.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t4,
LastSentAt: &t4,
},
@@ -2258,6 +2308,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -2275,6 +2326,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -2292,6 +2344,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t4,
EndsAt: t4,
FiredAt: t1,
LastEvaluationTime: t4,
LastSentAt: &t4,
ResolvedAt: &t4,
@@ -2311,6 +2364,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -2328,6 +2382,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -2345,6 +2400,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.NoData),
StartsAt: t1,
EndsAt: t4.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t4,
LastSentAt: &t4,
},
@@ -2458,6 +2514,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -2473,6 +2530,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t4,
EndsAt: t4.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t4,
LastSentAt: &t3,
},
@@ -2488,6 +2546,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t5, eval.NoData),
StartsAt: t5,
EndsAt: t5.Add(ResendDelay * 4),
FiredAt: t5,
LastEvaluationTime: t5,
LastSentAt: &t3,
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
@@ -2657,6 +2716,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.NoData),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -2761,6 +2821,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -2842,6 +2903,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -2873,6 +2935,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -2890,6 +2953,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -2908,6 +2972,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -2926,6 +2991,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -2945,6 +3011,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -2962,6 +3029,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -3009,6 +3077,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -3041,6 +3110,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -3104,6 +3174,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -3196,6 +3267,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -3277,6 +3349,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -3365,6 +3438,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
Values: map[string]float64{"A": float64(-1)},
@@ -3402,6 +3476,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
Values: map[string]float64{"A": float64(-1)},
@@ -3456,6 +3531,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
Values: map[string]float64{"A": float64(-1)},
@@ -3574,6 +3650,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t3,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -3714,6 +3791,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -3728,6 +3806,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t3,
LastSentAt: &t1,
},
@@ -3742,6 +3821,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.Normal),
StartsAt: t4,
EndsAt: t4,
FiredAt: t1,
LastEvaluationTime: t4,
ResolvedAt: &t4,
LastSentAt: &t4,
@@ -3920,6 +4000,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
Annotations: datasourceErrorAnnotations,
@@ -3936,6 +4017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: &t2,
},
@@ -3952,6 +4034,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t4, eval.Error),
StartsAt: t4,
EndsAt: t4.Add(ResendDelay * 4),
FiredAt: t4,
LastEvaluationTime: t4,
LastSentAt: &t2,
Annotations: datasourceErrorAnnotations,
@@ -4100,6 +4183,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -4201,6 +4285,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -4236,6 +4321,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -4321,6 +4407,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -4384,6 +4471,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t3,
},
@@ -4430,6 +4518,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -4534,6 +4623,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Error),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -4552,6 +4642,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t2,
LastSentAt: &t1,
},
@@ -350,6 +350,7 @@ func TestProcessEvalResults(t *testing.T) {
t2 := tn(2)
t3 := tn(3)
m := models.RuleMuts
baseRule := &models.AlertRule{
OrgID: 1,
@@ -475,6 +476,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
},
@@ -523,6 +525,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t2,
LastSentAt: &t2,
},
@@ -554,6 +557,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(4), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
FiredAt: tn(4),
LastEvaluationTime: tn(4),
LastSentAt: util.Pointer(tn(4)),
},
@@ -579,6 +583,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t2, eval.Normal),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t2,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -608,6 +613,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t2,
EndsAt: t2,
FiredAt: t1,
LastEvaluationTime: t3,
ResolvedAt: &t2,
LastSentAt: &t2,
@@ -640,6 +646,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(4), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: tn(4),
ResolvedAt: &t3,
LastSentAt: &t3,
@@ -706,6 +713,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: tn(4),
LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3.
},
@@ -959,6 +967,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(5), eval.Error),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
FiredAt: tn(5),
LastEvaluationTime: tn(5),
LastSentAt: util.Pointer(tn(5)),
},
@@ -1026,6 +1035,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4),
FiredAt: t3,
LastEvaluationTime: t3,
LastSentAt: &t1, // Resend delay is 30s, so last sent at is t1.
},
@@ -1057,6 +1067,7 @@ func TestProcessEvalResults(t *testing.T) {
}),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
Values: map[string]float64{
@@ -1092,6 +1103,7 @@ func TestProcessEvalResults(t *testing.T) {
}),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
FiredAt: t1,
LastEvaluationTime: t1,
LastSentAt: &t1,
Values: map[string]float64{
@@ -1124,6 +1136,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4),
FiredAt: t2,
LastEvaluationTime: t3,
LastSentAt: util.Pointer(t2),
},
@@ -1158,6 +1171,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(5), eval.Normal),
StartsAt: tn(5),
EndsAt: tn(5),
FiredAt: t2,
LastEvaluationTime: tn(5),
LastSentAt: util.Pointer(tn(5)),
ResolvedAt: util.Pointer(tn(5)),
@@ -1196,6 +1210,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(6), eval.Normal),
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4),
FiredAt: tn(4),
LastEvaluationTime: tn(6),
LastSentAt: util.Pointer(tn(5)),
},
@@ -1230,6 +1245,7 @@ func TestProcessEvalResults(t *testing.T) {
LatestResult: newEvaluation(tn(5), eval.Alerting),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
FiredAt: tn(5),
LastEvaluationTime: tn(5),
LastSentAt: util.Pointer(t3),
},
+8
View File
@@ -63,6 +63,9 @@ type State struct {
// conditions.
Values map[string]float64
// FiredAt is the time the state first transitions to Alerting.
FiredAt time.Time
StartsAt time.Time
// EndsAt is different from the Prometheus EndsAt as EndsAt is updated for both Normal states
// and states that have been resolved. It cannot be used to determine when a state was resolved.
@@ -128,6 +131,7 @@ func (a *State) Copy() *State {
Values: a.Values,
StartsAt: a.StartsAt,
EndsAt: a.EndsAt,
FiredAt: a.FiredAt,
ResolvedAt: a.ResolvedAt,
LastSentAt: a.LastSentAt,
LastEvaluationString: a.LastEvaluationString,
@@ -159,6 +163,9 @@ func (a *State) SetAlerting(reason string, startsAt, endsAt time.Time) {
a.StartsAt = startsAt
a.EndsAt = endsAt
a.Error = nil
// FiredAt is only ever set when the state is set to Alerting.
a.FiredAt = startsAt
}
// SetPending sets the state to Pending. It changes both the start and end time.
@@ -772,6 +779,7 @@ func patch(newState, existingState *State, result eval.Result) {
newState.LastEvaluationString = existingState.LastEvaluationString
newState.StartsAt = existingState.StartsAt
newState.EndsAt = existingState.EndsAt
newState.FiredAt = existingState.FiredAt
newState.ResolvedAt = existingState.ResolvedAt
newState.LastSentAt = existingState.LastSentAt
// Annotations can change over time, however we also want to maintain
+2
View File
@@ -46,6 +46,7 @@ func TestSetAlerting(t *testing.T) {
StateReason: "this is a reason",
StartsAt: mock.Now(),
EndsAt: mock.Now().Add(time.Minute),
FiredAt: mock.Now(),
},
}, {
name: "previous state is removed",
@@ -60,6 +61,7 @@ func TestSetAlerting(t *testing.T) {
State: eval.Alerting,
StartsAt: mock.Now(),
EndsAt: mock.Now().Add(time.Minute),
FiredAt: mock.Now(),
},
}}