public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add filters for health and contact point in Prometheus Rules api (#106580)

Browse Source 
This adds the ability to filter rules with the prometheus compatible api using:
1. `receiver_name` to filter by contact point name
2. `health` to filter by the health status of the rule (one of `ok`, `error`, `nodata`, or `unknown`)

This also ensures that groups with no rules (due to filters) are not returned.
This commit is contained in:
Moustafa Baiou
2025-06-17 16:57:03 +02:00
committed by GitHub
parent a302dad3c9
commit 0e6c7f84c3
3 changed files with 296 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+236 -6
View File
@@ -311,6 +311,13 @@ func withErrorState() forEachState {
}
}
func withNoDataState() forEachState {
return func(s *state.State) *state.State {
s.SetNoData("no data returned", timeNow(), timeNow().Add(5*time.Minute))
return s
}
}
func withLabels(labels data.Labels) forEachState {
return func(s *state.State) *state.State {
for k, v := range labels {
@@ -321,8 +328,6 @@ func withLabels(labels data.Labels) forEachState {
}
func TestRouteGetRuleStatuses(t *testing.T) {
// t.Skip() // TODO: Flaky test: https://github.com/grafana/grafana/issues/69146
timeNow = func() time.Time { return time.Date(2022, 3, 10, 14, 0, 0, 0, time.UTC) }
orgID := int64(1)
gen := ngmodels.RuleGen
@@ -1418,15 +1423,13 @@ func TestRouteGetRuleStatuses(t *testing.T) {
})
t.Run("test with filters on state", func(t *testing.T) {
t.Skip() // TODO: Flaky test: https://github.com/grafana/grafana/issues/69146
fakeStore, fakeAIM, api := setupAPI(t)
// create two rules in the same Rule Group to keep assertions simple
// create rules in the same Rule Group to keep assertions simple
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
})).GenerateManyRef(2)
})).GenerateManyRef(3)
// Need to sort these so we add alerts to the rules as ordered in the response
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
// The last two rules will have errors, however the first will be alerting
@@ -1576,6 +1579,182 @@ func TestRouteGetRuleStatuses(t *testing.T) {
// The error alert has been removed as the filters are inactive and firing
require.Len(t, rg.Rules[2].Alerts, 0)
})
t.Run("then with all rules filtered out, no groups returned", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=unknown", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
})
t.Run("test with filters on health", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
})).GenerateManyRef(4)
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
// Set health states
fakeStore.PutRule(context.Background(), rules...)
// create alert instances for each rule
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1)
fakeAIM.GenerateAlertInstances(orgID, rules[1].UID, 1, withAlertingErrorState())
fakeAIM.GenerateAlertInstances(orgID, rules[2].UID, 1, withErrorState())
fakeAIM.GenerateAlertInstances(orgID, rules[3].UID, 1, withNoDataState())
t.Run("invalid health returns 400 Bad Request", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=blah", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusBadRequest, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Contains(t, res.Error, "unknown health")
})
t.Run("first without filters", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 4)
})
t.Run("then with filter for ok health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=ok", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "ok", rg.Rules[0].Health)
})
t.Run("then with filter for error health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=error", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 2)
require.Equal(t, "error", rg.Rules[0].Health)
})
t.Run("then with filter for nodata health", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=nodata", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "nodata", rg.Rules[0].Health)
})
t.Run("then with multiple health filters", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=ok&health=error", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 3)
healths := []string{rg.Rules[0].Health, rg.Rules[1].Health}
require.ElementsMatch(t, healths, []string{"ok", "error"})
})
t.Run("then with all rules filtered out, no groups returned", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?health=unknown", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
})
t.Run("test with matcher on labels", func(t *testing.T) {
@@ -1750,6 +1929,57 @@ func TestRouteGetRuleStatuses(t *testing.T) {
require.Equal(t, map[string]int64{"normal": 1}, rg.Rules[0].TotalsFiltered)
})
})
t.Run("test with a contact point filter", func(t *testing.T) {
fakeStore, _, api := setupAPI(t)
rules := gen.With(gen.WithGroupKey(ngmodels.AlertRuleGroupKey{
NamespaceUID: "Folder-1",
RuleGroup: "Rule-Group-1",
OrgID: orgID,
}), gen.WithNotificationSettings(
ngmodels.NotificationSettings{
Receiver: "webhook-a",
GroupBy: []string{"alertname"},
},
)).GenerateManyRef(1)
fakeStore.PutRule(context.Background(), rules...)
t.Run("unknown receiver_name returns empty list", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?receiver_name=webhook-b", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 0)
})
t.Run("known receiver_name returns rules with that receiver", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?receiver_name=webhook-a", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
require.Len(t, rg.Rules, 1)
require.Equal(t, "webhook-a", rg.Rules[0].NotificationSettings.Receiver)
})
})
}
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv) {