public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Create annotation if Firing alert is removed (#45703) (#45865)

Browse Source 
This commit changes staleResultsHandler to create an annotation if the current state is Alerting and the result is being removed from the state cache as it has not been updated since 2x the evaluation interval.

(cherry picked from commit feae959c9d)
This commit is contained in:
George Robinson
2022-02-25 07:38:31 +01:00
committed by GitHub
parent c29c1691fd
commit cdf8bab022
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/state/manager.go
+6 -2
View File
@@ -186,7 +186,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
st.set(currentState)
if oldState != currentState.State {
go st.createAlertAnnotation(ctx, alertRule, currentState.Labels, result.EvaluatedAt, currentState.State, oldState)
go st.annotateState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, currentState.State, oldState)
}
return currentState
}
@@ -234,7 +234,7 @@ func translateInstanceState(state ngModels.InstanceStateType) eval.State {
}
}
func (st *Manager) createAlertAnnotation(ctx context.Context, alertRule *ngModels.AlertRule, labels data.Labels, evaluatedAt time.Time, state eval.State, previousState eval.State) {
func (st *Manager) annotateState(ctx context.Context, alertRule *ngModels.AlertRule, labels data.Labels, evaluatedAt time.Time, state eval.State, previousState eval.State) {
st.log.Debug("alert state changed creating annotation", "alertRuleUID", alertRule.UID, "newState", state.String(), "oldState", previousState.String())
labels = removePrivateLabels(labels)
@@ -297,6 +297,10 @@ func (st *Manager) staleResultsHandler(ctx context.Context, alertRule *ngModels.
if err = st.instanceStore.DeleteAlertInstance(ctx, s.OrgID, s.AlertRuleUID, labelsHash); err != nil {
st.log.Error("unable to delete stale instance from database", "error", err.Error(), "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheId)
}
if s.State == eval.Alerting {
st.annotateState(ctx, alertRule, s.Labels, time.Now(), eval.Normal, s.State)
}
}
}
}