Alerting: Add totalsFiltered to RuleResponse for hidden by filters count (#66883)

Alerting: Add totalsFiltered to RuleResponse to facilitate hidden by filters count

Currently, when both a limit_alerts and a matcher/state filter is applied, there is not enough information to determine how many alert instances were hidden by the filters. Only enough to determine the total hidden by the limit and filter combined.

This change adds a separate totalsFiltered field alongside the AlertRule totals that will contain the count of instances after filters but before limits.
This commit is contained in:
Matthew Jacobson
2023-04-21 09:35:12 +01:00
committed by GitHub
parent 29aaf429ad
commit eddd4f4508
3 changed files with 71 additions and 5 deletions
+10 -1
View File
@@ -329,13 +329,15 @@ func (srv PrometheusSrv) toRuleGroup(groupKey ngmodels.AlertRuleGroupKey, folder
states := srv.manager.GetStatesForRuleUID(rule.OrgID, rule.UID)
totals := make(map[string]int64)
totalsFiltered := make(map[string]int64)
for _, alertState := range states {
activeAt := alertState.StartsAt
valString := ""
if alertState.State == eval.Alerting || alertState.State == eval.Pending {
valString = formatValues(alertState)
}
totals[strings.ToLower(alertState.State.String())] += 1
stateKey := strings.ToLower(alertState.State.String())
totals[stateKey] += 1
// Do not add error twice when execution error state is Error
if alertState.Error != nil && rule.ExecErrState != ngmodels.ErrorErrState {
totals["error"] += 1
@@ -389,6 +391,12 @@ func (srv PrometheusSrv) toRuleGroup(groupKey ngmodels.AlertRuleGroupKey, folder
continue
}
totalsFiltered[stateKey] += 1
// Do not add error twice when execution error state is Error
if alertState.Error != nil && rule.ExecErrState != ngmodels.ErrorErrState {
totalsFiltered["error"] += 1
}
alertingRule.Alerts = append(alertingRule.Alerts, alert)
}
@@ -408,6 +416,7 @@ func (srv PrometheusSrv) toRuleGroup(groupKey ngmodels.AlertRuleGroupKey, folder
alertingRule.Rule = newRule
alertingRule.Totals = totals
alertingRule.TotalsFiltered = totalsFiltered
newGroup.Rules = append(newGroup.Rules, alertingRule)
newGroup.Interval = float64(rule.IntervalSeconds)
// TODO yuri. Change that when scheduler will process alerts in groups