public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add filters for health and contact point in Prometheus Rules api (#106580)

Browse Source 
This adds the ability to filter rules with the prometheus compatible api using:
1. `receiver_name` to filter by contact point name
2. `health` to filter by the health status of the rule (one of `ok`, `error`, `nodata`, or `unknown`)

This also ensures that groups with no rules (due to filters) are not returned.
This commit is contained in:
Moustafa Baiou
2025-06-17 16:57:03 +02:00
committed by GitHub
parent a302dad3c9
commit 0e6c7f84c3
3 changed files with 296 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+236 -6
View File
@@ -311,6 +311,13 @@ func withErrorState() forEachState {
}
}
func withNoDataState() forEachState {
return func(s *state.State) *state.State {
s.SetNoData("no data returned", timeNow(), timeNow().Add(5*time.Minute))
return s
}
}
func withLabels(labels data.Labels) forEachState {
return func(s *state.State) *state.State {
for k, v := range labels {
@@ -321,8 +328,6 @@ func withLabels(labels data.Labels) forEachState {
}
func TestRouteGetRuleStatuses(t *testing.T) {
// t.Skip() // TODO: Flaky test: https://github.com/grafana/grafana/issues/69146
timeNow = func() time.Time { return time.Date(2022, 3, 10, 14, 0, 0, 0, time.UTC) }
orgID := int64(1)
gen := ngmodels.RuleGen
@@ -1418,15 +1423,13 @@ func TestRouteGetRuleStatuses(t *testing.T) {
})
t.Run("test with filters on state", func(t *testing.T) {
t.Skip() // TODO: Flaky test: https://github.com/grafana/grafana/issues/69146
fakeStore, fakeAIM, api := setupAPI(t)
// create two rules in the same Rule Group to keep assertions simple
// create rules in the same Rule Group to keep assertions simple
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
})).GenerateManyRef(2)
})).GenerateManyRef(3)
// Need to sort these so we add alerts to the rules as ordered in the response
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
// The last two rules will have errors, however the first will be alerting
@@ -1576,6 +1579,182 @@ func TestRouteGetRuleStatuses(t *testing.T) {
// The error alert has been removed as the filters are inactive and firing
require.Len(t, rg.Rules[2].Alerts, 0)
})
t.Run("then with all rules filtered out, no groups returned", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=unknown", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
})
t.Run("test with filters on health", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
})).GenerateManyRef(4)
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
// Set health states
fakeStore.PutRule(context.Background(), rules...)
// create alert instances for each rule
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1)
fakeAIM.GenerateAlertInstances(orgID, rules[1].UID, 1, withAlertingErrorState())
fakeAIM.GenerateAlertInstances(orgID, rules[2].UID, 1, withErrorState())
fakeAIM.GenerateAlertInstances(orgID, rules[3].UID, 1, withNoDataState())
t.Run("invalid health returns 400 Bad Request", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=blah", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusBadRequest, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Contains(t, res.Error, "unknown health")
})
t.Run("first without filters", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 4)
})
t.Run("then with filter for ok health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=ok", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "ok", rg.Rules[0].Health)
})
t.Run("then with filter for error health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=error", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 2)
require.Equal(t, "error", rg.Rules[0].Health)
})
t.Run("then with filter for nodata health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=nodata", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "nodata", rg.Rules[0].Health)
})
t.Run("then with multiple health filters", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=ok&health=error", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 3)
healths := []string{rg.Rules[0].Health, rg.Rules[1].Health}
require.ElementsMatch(t, healths, []string{"ok", "error"})
})
t.Run("then with all rules filtered out, no groups returned", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=unknown", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
})
t.Run("test with matcher on labels", func(t *testing.T) {
@@ -1750,6 +1929,57 @@ func TestRouteGetRuleStatuses(t *testing.T) {
require.Equal(t, map[string]int64{"normal": 1}, rg.Rules[0].TotalsFiltered)
})
})
t.Run("test with a contact point filter", func(t *testing.T) {
fakeStore, _, api := setupAPI(t)
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
}), gen.WithNotificationSettings(
ngmodels.NotificationSettings{
Receiver: "webhook-a",
GroupBy: []string{"alertname"},
},
)).GenerateManyRef(1)
fakeStore.PutRule(context.Background(), rules...)
t.Run("unknown receiver_name returns empty list", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?receiver_name=webhook-b", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
t.Run("known receiver_name returns rules with that receiver", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?receiver_name=webhook-a", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "webhook-a", rg.Rules[0].NotificationSettings.Receiver)
})
})
}
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv) {
pkg/services/ngalert/api/prometheus/api_prometheus.go
+56 -17
View File
@@ -189,24 +189,23 @@ func getMatchersFromQuery(v url.Values) (labels.Matchers, error) {
return matchers, nil
}
func getStatesFromQuery(v url.Values) ([]eval.State, error) {
var states []eval.State
func getStatesFromQuery(v url.Values) (map[eval.State]struct{}, error) {
states := make(map[eval.State]struct{})
for _, s := range v["state"] {
s = strings.ToLower(s)
switch s {
case "normal", "inactive":
states = append(states, eval.Normal)
states[eval.Normal] = struct{}{}
case "alerting", "firing":
states = append(states, eval.Alerting)
states[eval.Alerting] = struct{}{}
case "pending":
states = append(states, eval.Pending)
states[eval.Pending] = struct{}{}
case "nodata":
states = append(states, eval.NoData)
// nolint:goconst
states[eval.NoData] = struct{}{}
case "error":
states = append(states, eval.Error)
states[eval.Error] = struct{}{}
case "recovering":
states = append(states, eval.Recovering)
states[eval.Recovering] = struct{}{}
default:
return states, fmt.Errorf("unknown state '%s'", s)
}
@@ -214,6 +213,20 @@ func getStatesFromQuery(v url.Values) ([]eval.State, error) {
return states, nil
}
func getHealthFromQuery(v url.Values) (map[string]struct{}, error) {
health := make(map[string]struct{})
for _, s := range v["health"] {
s = strings.ToLower(s)
switch s {
case "ok", "error", "nodata", "unknown":
health[s] = struct{}{}
default:
return nil, fmt.Errorf("unknown health '%s'", s)
}
}
return health, nil
}
type RuleGroupStatusesOptions struct {
Ctx context.Context
OrgID int64
@@ -400,16 +413,20 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
stateFilter, err := getStatesFromQuery(opts.Query)
stateFilterSet, err := getStatesFromQuery(opts.Query)
if err != nil {
ruleResponse.Status = "error"
ruleResponse.Error = err.Error()
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
stateFilterSet := make(map[eval.State]struct{})
for _, state := range stateFilter {
stateFilterSet[state] = struct{}{}
healthFilterSet, err := getHealthFromQuery(opts.Query)
if err != nil {
ruleResponse.Status = "error"
ruleResponse.Error = err.Error()
ruleResponse.ErrorType = apiv1.ErrBadData
return ruleResponse
}
var labelOptions []ngmodels.LabelOption
@@ -436,12 +453,15 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
ruleGroups := opts.Query["rule_group"]
receiverName := opts.Query.Get("receiver_name")
alertRuleQuery := ngmodels.ListAlertRulesQuery{
OrgID: opts.OrgID,
NamespaceUIDs: namespaceUIDs,
DashboardUID: dashboardUID,
PanelID: panelID,
RuleGroups: ruleGroups,
ReceiverName: receiverName,
}
ruleList, err := store.ListAlertRules(opts.Ctx, &alertRuleQuery)
if err != nil {
@@ -488,15 +508,21 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru
rulesTotals[k] += v
}
if len(stateFilter) > 0 {
filterRules(ruleGroup, stateFilterSet)
if len(stateFilterSet) > 0 {
filterRulesByState(ruleGroup, stateFilterSet)
}
if len(healthFilterSet) > 0 {
filterRulesByHealth(ruleGroup, healthFilterSet)
}
if limitRulesPerGroup > -1 && int64(len(ruleGroup.Rules)) > limitRulesPerGroup {
ruleGroup.Rules = ruleGroup.Rules[0:limitRulesPerGroup]
}
ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
if len(ruleGroup.Rules) > 0 {
ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
}
}
ruleResponse.Data.NextToken = newToken
@@ -578,7 +604,7 @@ func getGroupedRules(log log.Logger, ruleList ngmodels.RulesGroup, ruleNamesSet
return ruleGroups
}
func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]struct{}) {
func filterRulesByState(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]struct{}) {
// Filtering is weird but firing, pending, and normal filters also need to be
// applied to the rule. Others such as nodata and error should have no effect.
// This is to match the current behavior in the UI.
@@ -604,6 +630,19 @@ func filterRules(ruleGroup *apimodels.RuleGroup, withStatesFast map[eval.State]s
ruleGroup.Rules = filteredRules
}
func filterRulesByHealth(ruleGroup *apimodels.RuleGroup, withHealthFast map[string]struct{}) {
// Filtering is weird but error and nodata filters also need to be
// applied to the rule. Others such as firing, pending, and normal should have no effect.
// This is to match the current behavior in the UI.
filteredRules := make([]apimodels.AlertingRule, 0, len(ruleGroup.Rules))
for _, rule := range ruleGroup.Rules {
if _, ok := withHealthFast[rule.Health]; ok {
filteredRules = append(filteredRules, rule)
}
}
ruleGroup.Rules = filteredRules
}
// This is the same as matchers.Matches but avoids the need to create a LabelSet
func matchersMatch(matchers []*labels.Matcher, labels map[string]string) bool {
for _, m := range matchers {
pkg/services/ngalert/tests/fakes/rules.go
+4
View File
@@ -228,6 +228,10 @@ func (f *RuleStore) ListAlertRules(_ context.Context, q *models.ListAlertRulesQu
}
}
if q.ReceiverName != "" && (len(r.NotificationSettings) < 1 || r.NotificationSettings[0].Receiver != q.ReceiverName) {
continue
}
ruleList = append(ruleList, r)
}