Alerting: Add filters for health and contact point in Prometheus Rules api (#106580)

This adds the ability to filter rules with the prometheus compatible api using:
1. `receiver_name` to filter by contact point name
2. `health` to filter by the health status of the rule (one of `ok`, `error`, `nodata`, or `unknown`)

This also ensures that groups with no rules (due to filters) are not returned.
This commit is contained in:
Moustafa Baiou
2025-06-17 16:57:03 +02:00
committed by GitHub
parent a302dad3c9
commit 0e6c7f84c3
3 changed files with 296 additions and 23 deletions
@@ -189,24 +189,23 @@ func getMatchersFromQuery(v url.Values) (labels.Matchers, error) {
return matchers, nil
}
func getStatesFromQuery(v url.Values) ([]eval.State, error) {
var states []eval.State
func getStatesFromQuery(v url.Values) (map[eval.State]struct{}, error) {
states := make(map[eval.State]struct{})
for _, s := range v["state"] {
s = strings.ToLower(s)
switch s {
case "normal", "inactive":
states = append(states, eval.Normal)
states[eval.Normal] = struct{}{}
case "alerting", "firing":
states = append(states, eval.Alerting)
states[eval.Alerting] = struct{}{}
case "pending":
states = append(states, eval.Pending)
states[eval.Pending] = struct{}{}
case "nodata":
states = append(states, eval.NoData)
// nolint:goconst
states[eval.NoData] = struct{}{}
case "error":
states = append(states, eval.Error)
states[eval.Error] = struct{}{}
case "recovering":
states = append(states, eval.Recovering)
states[eval.Recovering] = struct{}{}
default:
return states, fmt.Errorf("unknown state '%s'", s)
}
@@ -214,6 +213,20 @@ func getStatesFromQuery(v url.Values) ([]eval.State, error) {
return states, nil
}
func getHealthFromQuery(v url.Values) (map[string]struct{}, error) {
health := make(map[string]struct{})
for _, s := range v["health"] {
s = strings.ToLower(s)
switch s {
case "ok", "error", "nodata", "unknown":
health[s] = struct{}{}
default:
return nil, fmt.Errorf("unknown health '%s'", s)
}
}
return health, nil
}
type RuleGroupStatusesOptions struct {
Ctx context.Context
OrgID int64
@@ -400,16 +413,20 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
stateFilter, err := getStatesFromQuery(opts.Query)
stateFilterSet, err := getStatesFromQuery(opts.Query)
if err != nil {
ruleResponse.Status = "error"
ruleResponse.Error = err.Error()
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
stateFilterSet := make(map[eval.State]struct{})
for _, state := range stateFilter {
stateFilterSet[state] = struct{}{}
healthFilterSet, err := getHealthFromQuery(opts.Query)
if err != nil {
ruleResponse.Status = "error"
ruleResponse.Error = err.Error()
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
var labelOptions []ngmodels.LabelOption
@@ -436,12 +453,15 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
ruleGroups := opts.Query["rule_group"]
receiverName := opts.Query.Get("receiver_name")
alertRuleQuery := ngmodels.ListAlertRulesQuery{
OrgID: opts.OrgID,
NamespaceUIDs: namespaceUIDs,
DashboardUID: dashboardUID,
PanelID: panelID,
RuleGroups: ruleGroups,
ReceiverName: receiverName,
}
ruleList, err := store.ListAlertRules(opts.Ctx, &alertRuleQuery)
if err != nil {
@@ -488,15 +508,21 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
rulesTotals[k] += v
}
if len(stateFilter) > 0 {
filterRules(ruleGroup, stateFilterSet)
if len(stateFilterSet) > 0 {
filterRulesByState(ruleGroup, stateFilterSet)
}
if len(healthFilterSet) > 0 {
filterRulesByHealth(ruleGroup, healthFilterSet)
}
if limitRulesPerGroup > -1 && int64(len(ruleGroup.Rules)) > limitRulesPerGroup {
ruleGroup.Rules = ruleGroup.Rules[0:limitRulesPerGroup]
}
ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
if len(ruleGroup.Rules) > 0 {
ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
}
}
ruleResponse.Data.NextToken = newToken
@@ -578,7 +604,7 @@ func getGroupedRules(log log.Logger, ruleList ngmodels.RulesGroup, ruleNamesSet
return ruleGroups
}
func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]struct{}) {
func filterRulesByState(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]struct{}) {
// Filtering is weird but firing, pending, and normal filters also need to be
// applied to the rule. Others such as nodata and error should have no effect.
// This is to match the current behavior in the UI.
@@ -604,6 +630,19 @@ func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]s
ruleGroup.Rules = filteredRules
}
func filterRulesByHealth(ruleGroup *apimodels.RuleGroup, withHealthFast map[string]struct{}) {
// Filtering is weird but error and nodata filters also need to be
// applied to the rule. Others such as firing, pending, and normal should have no effect.
// This is to match the current behavior in the UI.
filteredRules := make([]apimodels.AlertingRule, 0, len(ruleGroup.Rules))
for _, rule := range ruleGroup.Rules {
if _, ok := withHealthFast[rule.Health]; ok {
filteredRules = append(filteredRules, rule)
}
}
ruleGroup.Rules = filteredRules
}
// This is the same as matchers.Matches but avoids the need to create a LabelSet
func matchersMatch(matchers []*labels.Matcher, labels map[string]string) bool {
for _, m := range matchers {