From eae77aa695630d2bc76da9cfc43d8072cd866c37 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 29 May 2025 23:22:35 +0200 Subject: [PATCH] Alerting: Resend alerts for states that are missing in the eval results (#105965) What is this feature? This PR fixes the MissingSeriesEvalsToResolve behavior when it's set to more than 4 evaluation intervals. Why do we need this feature? The MissingSeriesEvalsToResolve setting was not working correctly due to alerts being auto-resolved by Alertmanager after 4 evaluation intervals (via the endsAt field). Before we had deleteStaleStatesFromCache method that was returning only stale states that had to be resolved. Non-stale states for which the current evaluation does not have a series never had endsAt updated and were never resend to the Alertmanager, so they were automatically resolved after 4 evaluations regardless of the setting. The new processMissingSeriesStates returns state for each missing series on every evaluation, and resolves the stale ones. This guarantees that alerts without series still alert for the configured number of evaluations. --- pkg/services/ngalert/state/cache.go | 12 +- pkg/services/ngalert/state/manager.go | 71 ++-- .../ngalert/state/manager_private_test.go | 305 ++++++++++++++++++ pkg/services/ngalert/state/manager_test.go | 5 +- pkg/services/ngalert/state/state.go | 8 +- pkg/services/ngalert/state/state_test.go | 2 +- 6 files changed, 363 insertions(+), 40 deletions(-) diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 24b40061fa4..4c5dfd35e33 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -167,25 +167,23 @@ func expand(ctx context.Context, log log.Logger, name string, original map[strin return expanded, errs } -func (rs *ruleStates) deleteStates(predicate func(s *State) bool) []*State { - deleted := make([]*State, 0) +func (rs *ruleStates) deleteStates(predicate func(s *State) bool) { for id, state := range rs.states { if predicate(state) { delete(rs.states, id) - deleted = append(deleted, state) } } - return deleted } -func (c *cache) deleteRuleStates(ruleKey ngModels.AlertRuleKey, predicate func(s *State) bool) []*State { +// deleteRuleStates iterates over all states for the given rule and deletes those where predicate returns true. +// The predicate function is called once for each state and should return true if the state should be deleted. +func (c *cache) deleteRuleStates(ruleKey ngModels.AlertRuleKey, predicate func(s *State) bool) { c.mtxStates.Lock() defer c.mtxStates.Unlock() ruleStates, ok := c.states[ruleKey.OrgID][ruleKey.UID] if ok { - return ruleStates.deleteStates(predicate) + ruleStates.deleteStates(predicate) } - return nil } func (c *cache) setRuleStates(ruleKey ngModels.AlertRuleKey, s ruleStates) { diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 5a1a01c2af4..782b871393a 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -352,13 +352,13 @@ func (st *Manager) ProcessEvalResults( logger.Debug("State manager processing evaluation results", "resultCount", len(results)) states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger, fn, evaluatedAt) - staleStates := st.deleteStaleStatesFromCache(logger, evaluatedAt, alertRule, fn) + missingSeriesStates, staleCount := st.processMissingSeriesStates(logger, evaluatedAt, alertRule, states, fn) span.AddEvent("results processed", trace.WithAttributes( attribute.Int64("state_transitions", int64(len(states))), - attribute.Int64("stale_states", int64(len(staleStates))), + attribute.Int64("stale_states", staleCount), )) - allChanges := StateTransitions(append(states, staleStates...)) + allChanges := StateTransitions(append(states, missingSeriesStates...)) // It's important that this is done *before* we sync the states to the persister. Otherwise, we will not persist // the LastSentAt field to the store. @@ -386,7 +386,7 @@ func (st *Manager) ProcessEvalResults( func (st *Manager) updateLastSentAt(states StateTransitions, evaluatedAt time.Time) StateTransitions { var result StateTransitions for _, t := range states { - if t.NeedsSending(st.ResendDelay, st.ResolvedRetention) { + if t.NeedsSending(evaluatedAt, st.ResendDelay, st.ResolvedRetention) { t.LastSentAt = &evaluatedAt result = append(result, t) } @@ -517,31 +517,47 @@ func translateInstanceState(state ngModels.InstanceStateType) eval.State { } } -func (st *Manager) deleteStaleStatesFromCache(logger log.Logger, evaluatedAt time.Time, alertRule *ngModels.AlertRule, takeImageFn takeImageFn) []StateTransition { - // If we are removing two or more stale series it makes sense to share the resolved image as the alert rule is the same. - // TODO: We will need to change this when we support images without screenshots as each series will have a different image - staleStates := st.cache.deleteRuleStates(alertRule.GetKey(), func(s *State) bool { - return stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds, alertRule.GetMissingSeriesEvalsToResolve()) - }) - resolvedStates := make([]StateTransition, 0, len(staleStates)) +// processMissingSeriesStates receives the updated state transitions +// that we got from the alert rule, and checks the cache for any states +// that are not in the current evaluation. The missing states are +// for series that are no longer present in the current evaluation. +// For each missing state, we check if it is stale, and if so, we resolve it. +// At the end we return the missing states so that later they can be sent +// to the alertmanager if needed. +func (st *Manager) processMissingSeriesStates(logger log.Logger, evaluatedAt time.Time, alertRule *ngModels.AlertRule, evalTransitions []StateTransition, takeImageFn takeImageFn) ([]StateTransition, int64) { + missingTransitions := []StateTransition{} + var staleStatesCount int64 = 0 - for _, s := range staleStates { - logger.Info("Detected stale state entry", "cacheID", s.CacheID, "state", s.State, "reason", s.StateReason) + st.cache.deleteRuleStates(alertRule.GetKey(), func(s *State) bool { + // We need only states that are not present in the current evaluation, so + // skip the state if it was just evaluated. + if s.LastEvaluationTime.Equal(evaluatedAt) { + return false + } + // After this point, we know that the state is not in the current evaluation. + // Now we need check if it's stale, and if so, we need to resolve it. oldState := s.State oldReason := s.StateReason + isStale := stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds, alertRule.GetMissingSeriesEvalsToResolve()) - s.State = eval.Normal - s.StateReason = ngModels.StateReasonMissingSeries - s.EndsAt = evaluatedAt - s.LastEvaluationTime = evaluatedAt + if isStale { + logger.Info("Detected stale state entry", "cacheID", s.CacheID, "state", s.State, "reason", s.StateReason) - // By setting ResolvedAt we trigger the scheduler to send a resolved notification to the Alertmanager. - if s.ShouldBeResolved(oldState) { - s.ResolvedAt = &evaluatedAt - image := takeImageFn("stale state") - if image != nil { - s.Image = image + s.State = eval.Normal + s.StateReason = ngModels.StateReasonMissingSeries + s.LastEvaluationTime = evaluatedAt + s.EndsAt = evaluatedAt + + // By setting ResolvedAt we trigger the scheduler to send a resolved notification to the Alertmanager. + if s.ShouldBeResolved(oldState) { + s.ResolvedAt = &evaluatedAt + image := takeImageFn("stale state") + if image != nil { + s.Image = image + } } + + staleStatesCount++ } record := StateTransition{ @@ -549,9 +565,12 @@ func (st *Manager) deleteStaleStatesFromCache(logger log.Logger, evaluatedAt tim PreviousState: oldState, PreviousStateReason: oldReason, } - resolvedStates = append(resolvedStates, record) - } - return resolvedStates + missingTransitions = append(missingTransitions, record) + + return isStale + }) + + return missingTransitions, staleStatesCount } // stateIsStale determines whether the evaluation state is considered stale. diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 1e56070455c..484888b58f2 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -657,6 +657,30 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[time.Time][]StateTransition{ t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluation(t1, eval.Normal), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1024,6 +1048,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1089,6 +1126,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[time.Time][]StateTransition{ t3: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + }, + }, { PreviousState: eval.NoData, State: &State{ @@ -1468,6 +1518,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluationWithValues(t1, eval.Normal, map[string]float64{"A": 1}), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1}, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1485,6 +1548,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, ngmodels.Alerting: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluationWithValues(t1, eval.Normal, map[string]float64{"A": 1}), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1}, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1505,6 +1581,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, ngmodels.OK: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluationWithValues(t1, eval.Normal, map[string]float64{"A": 1}), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1}, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1523,6 +1612,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, ngmodels.KeepLast: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluationWithValues(t1, eval.Normal, map[string]float64{"A": 1}), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1}, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1558,6 +1660,32 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluation(t1, eval.Normal), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1615,6 +1743,30 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, ngmodels.Alerting: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluation(t1, eval.Normal), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -1831,6 +1983,30 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluation(t1, eval.Normal), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Pending, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -2092,6 +2268,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t3: { + { + PreviousState: eval.NoData, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + LatestResult: newEvaluation(t2, eval.NoData), + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + LastSentAt: &t2, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Pending, State: &State{ @@ -2181,6 +2370,20 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -2460,6 +2663,20 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, t4: { + { + PreviousState: eval.NoData, + State: &State{ + Labels: labels["system + rule + no-data"], + Annotations: baseRule.Annotations, + State: eval.NoData, + LatestResult: newEvaluation(t3, eval.NoData), + StartsAt: t2, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + LastSentAt: &t2, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -2486,6 +2703,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, t5: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + Annotations: baseRule.Annotations, + State: eval.Normal, + LatestResult: newEvaluation(t4, eval.Normal), + StartsAt: t4, + EndsAt: t4, + LastEvaluationTime: t4, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.NoData, State: &State{ @@ -2795,6 +3025,18 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + LatestResult: newEvaluation(t1, eval.Normal), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -2880,6 +3122,20 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + FiredAt: &t1, + LastEvaluationTime: t1, + LastSentAt: &t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -3055,6 +3311,18 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + State: eval.Pending, + LatestResult: newEvaluation(t1, eval.Alerting), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t1, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -3069,6 +3337,19 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, t3: { + { + PreviousState: eval.NoData, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + LatestResult: newEvaluation(t2, eval.NoData), + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + LastSentAt: &t2, + EvaluationDuration: time.Millisecond * 10, + }, + }, { PreviousState: eval.Pending, State: &State{ @@ -3406,6 +3687,18 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.ErrorErrState: { t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Pending, + LatestResult: newEvaluationWithValues(t1, eval.Alerting, map[string]float64{"A": 1.0}), + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1.0}, + }, + }, { PreviousState: eval.Normal, State: &State{ @@ -3499,6 +3792,18 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.ErrorErrState: { t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + LatestResult: newEvaluationWithValues(t1, eval.Normal, map[string]float64{"A": 1.0}), + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + Values: map[string]float64{"A": 1.0}, + }, + }, { PreviousState: eval.Normal, State: &State{ diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 050bb2e37f7..a2a06b6c2c3 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -1417,9 +1417,10 @@ func TestProcessEvalResults(t *testing.T) { statePersister := state.NewSyncStatePersisiter(log.New("ngalert.state.manager.persist"), cfg) st := state.NewManager(cfg, statePersister) rule := models.RuleGen.GenerateRef() - var results = eval.GenerateResults(rand.Intn(4)+1, eval.ResultGen(eval.WithEvaluatedAt(clk.Now()))) + now := clk.Now() + var results = eval.GenerateResults(rand.Intn(4)+1, eval.ResultGen(eval.WithEvaluatedAt(now))) - states := st.ProcessEvalResults(context.Background(), clk.Now(), rule, results, make(data.Labels), nil) + states := st.ProcessEvalResults(context.Background(), now, rule, results, make(data.Labels), nil) require.NotEmpty(t, states) savedStates := make(map[data.Fingerprint]models.AlertInstance) diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index f1cdaeecae1..a393d95cabb 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -581,7 +581,7 @@ func resultKeepLast(state *State, rule *models.AlertRule, result eval.Result, lo // - The state has been resolved since the last notification. // - The state is firing and the last notification was sent at least resendDelay ago. // - The state was resolved within the resolvedRetention period, and the last notification was sent at least resendDelay ago. -func (a *State) NeedsSending(resendDelay time.Duration, resolvedRetention time.Duration) bool { +func (a *State) NeedsSending(now time.Time, resendDelay time.Duration, resolvedRetention time.Duration) bool { if a.State == eval.Pending { // We do not send notifications for pending states. return false @@ -594,13 +594,13 @@ func (a *State) NeedsSending(resendDelay time.Duration, resolvedRetention time.D // For normal states, we should only be sending if this is a resolved notification or a re-send of the resolved // notification within the resolvedRetention period. - if a.State == eval.Normal && (a.ResolvedAt == nil || a.LastEvaluationTime.Sub(*a.ResolvedAt) > resolvedRetention) { + if a.State == eval.Normal && (a.ResolvedAt == nil || now.Sub(*a.ResolvedAt) > resolvedRetention) { return false } - // We should send, and re-send notifications, each time LastSentAt is <= LastEvaluationTime + resendDelay. + // We should send, and re-send notifications, each time LastSentAt is <= now + resendDelay. // This can include normal->normal transitions that were resolved in recent past evaluations. - return a.LastSentAt == nil || !a.LastSentAt.Add(resendDelay).After(a.LastEvaluationTime) + return a.LastSentAt == nil || !a.LastSentAt.Add(resendDelay).After(now) } func (a *State) Equals(b *State) bool { diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index fb2b5f4bd68..17e6733a0d7 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -512,7 +512,7 @@ func TestNeedsSending(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expected, tc.testState.NeedsSending(tc.resendDelay, tc.resolvedRetention)) + assert.Equal(t, tc.expected, tc.testState.NeedsSending(evaluationTime, tc.resendDelay, tc.resolvedRetention)) }) } }