From cdf8bab0229db0113fe181a9a11e8890c50f51a7 Mon Sep 17 00:00:00 2001 From: George Robinson Date: Fri, 25 Feb 2022 06:38:31 +0000 Subject: [PATCH] Alerting: Create annotation if Firing alert is removed (#45703) (#45865) This commit changes staleResultsHandler to create an annotation if the current state is Alerting and the result is being removed from the state cache as it has not been updated since 2x the evaluation interval. (cherry picked from commit feae959c9da12885fc79786d57d7c415595c260a) --- pkg/services/ngalert/state/manager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index c476547518f..bb8e30d33e4 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -186,7 +186,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu st.set(currentState) if oldState != currentState.State { - go st.createAlertAnnotation(ctx, alertRule, currentState.Labels, result.EvaluatedAt, currentState.State, oldState) + go st.annotateState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, currentState.State, oldState) } return currentState } @@ -234,7 +234,7 @@ func translateInstanceState(state ngModels.InstanceStateType) eval.State { } } -func (st *Manager) createAlertAnnotation(ctx context.Context, alertRule *ngModels.AlertRule, labels data.Labels, evaluatedAt time.Time, state eval.State, previousState eval.State) { +func (st *Manager) annotateState(ctx context.Context, alertRule *ngModels.AlertRule, labels data.Labels, evaluatedAt time.Time, state eval.State, previousState eval.State) { st.log.Debug("alert state changed creating annotation", "alertRuleUID", alertRule.UID, "newState", state.String(), "oldState", previousState.String()) labels = removePrivateLabels(labels) @@ -297,6 +297,10 @@ func (st *Manager) staleResultsHandler(ctx context.Context, alertRule *ngModels. if err = st.instanceStore.DeleteAlertInstance(ctx, s.OrgID, s.AlertRuleUID, labelsHash); err != nil { st.log.Error("unable to delete stale instance from database", "error", err.Error(), "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheId) } + + if s.State == eval.Alerting { + st.annotateState(ctx, alertRule, s.Labels, time.Now(), eval.Normal, s.State) + } } } }