public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add limits to the Prometheus Rules API (#65169)

Browse Source 
This commit adds a number of limits to the Grafana flavor of the
Prometheus Rules API:

1. `limit` limits the maximum number of Rule Groups returned
2. `limit_rules` limits the maximum number of rules per Rule Group
3. `limit_alerts` limits the maximum number of alerts per rule

It sorts Rule Groups and rules within Rule Groups such that data in the
response is stable across requests. It also returns summaries (totals) for
all Rule Groups, individual Rule Groups and rules.
This commit is contained in:
George Robinson
2023-04-03 10:17:02 +01:00
committed by GitHub
parent 093498a03b
commit d96b0a71d3
8 changed files with 613 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+208 -3
View File
@@ -305,6 +305,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"activeAt": "0001-01-01T00:00:00Z",
"value": ""
}],
"totals": {
"normal": 1
},
"labels": {
"__a_private_label_on_the_rule__": "a_value"
},
@@ -314,10 +317,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"duration": 180,
"evaluationTime": 60
}],
"totals": {
"inactive": 1
},
"interval": 60,
"lastEvaluation": "2022-03-10T14:01:00Z",
"evaluationTime": 60
}]
}],
"totals": {
"inactive": 1
}
}
}
`, folder.Title), string(r.Body()))
@@ -358,6 +367,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"activeAt": "0001-01-01T00:00:00Z",
"value": ""
}],
"totals": {
"normal": 1
},
"labels": {
"__a_private_label_on_the_rule__": "a_value",
"__alert_rule_uid__": "RuleUID"
@@ -368,10 +380,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"duration": 180,
"evaluationTime": 60
}],
"totals": {
"inactive": 1
},
"interval": 60,
"lastEvaluation": "2022-03-10T14:01:00Z",
"evaluationTime": 60
}]
}],
"totals": {
"inactive": 1
}
}
}
`, folder.Title), string(r.Body()))
@@ -406,6 +424,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"activeAt": "0001-01-01T00:00:00Z",
"value": ""
}],
"totals": {
"normal": 1
},
"labels": {
"__a_private_label_on_the_rule__": "a_value"
},
@@ -415,10 +436,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
"duration": 180,
"evaluationTime": 60
}],
"totals": {
"inactive": 1
},
"interval": 60,
"lastEvaluation": "2022-03-10T14:01:00Z",
"evaluationTime": 60
}]
}],
"totals": {
"inactive": 1
}
}
}
`, folder.Title), string(r.Body()))
@@ -504,6 +531,184 @@ func TestRouteGetRuleStatuses(t *testing.T) {
assert.Emptyf(t, rules, "not all expected rules were returned")
})
})
t.Run("test with limit on Rule Groups", func(t *testing.T) {
fakeStore, _, _, api := setupAPI(t)
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID)))
fakeStore.PutRule(context.Background(), rules...)
t.Run("first without limit", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 inactive rules across all Rule Groups
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 2)
for _, rg := range res.Data.RuleGroups {
// Each Rule Group should have 1 inactive rule
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
}
})
t.Run("then with limit", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 inactive rules across all Rule Groups
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
// The Rule Group within the limit should have 1 inactive rule
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
})
})
t.Run("test with limit rules", func(t *testing.T) {
fakeStore, _, _, api := setupAPI(t)
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1")))
fakeStore.PutRule(context.Background(), rules...)
t.Run("first without limit", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 inactive rules across all Rule Groups
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 2)
for _, rg := range res.Data.RuleGroups {
// Each Rule Group should have 1 inactive rule
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
}
})
t.Run("then with limit", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=1", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 inactive rules
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 1)
rg := res.Data.RuleGroups[0]
// The Rule Group within the limit should have 1 inactive rule because of the limit
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
})
})
t.Run("test with limit alerts", func(t *testing.T) {
fakeStore, fakeAIM, _, api := setupAPI(t)
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1")))
fakeStore.PutRule(context.Background(), rules...)
// create a normal and firing alert for each rule
for _, r := range rules {
fakeAIM.GenerateAlertInstances(orgID, r.UID, 1)
fakeAIM.GenerateAlertInstances(orgID, r.UID, 1, withAlertingState())
}
t.Run("first without limit", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 firing rules across all Rule Groups
require.Equal(t, map[string]int64{"firing": 2}, res.Data.Totals)
require.Len(t, res.Data.RuleGroups, 2)
for _, rg := range res.Data.RuleGroups {
// Each Rule Group should have 1 firing rule
require.Equal(t, map[string]int64{"firing": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
// Each rule should have two alerts
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rg.Rules[0].Totals)
}
})
t.Run("then with limits", func(t *testing.T) {
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=1&limit_alerts=1", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: r},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
OrgRole: org.RoleViewer,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// There should be 2 firing rules across all Rule Groups
require.Equal(t, map[string]int64{"firing": 2}, res.Data.Totals)
rg := res.Data.RuleGroups[0]
// The Rule Group within the limit should have 1 inactive rule because of the limit
require.Equal(t, map[string]int64{"firing": 1}, rg.Totals)
require.Len(t, rg.Rules, 1)
rule := rg.Rules[0]
// The rule should have two alerts, but just one should be returned
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rule.Totals)
require.Len(t, rule.Alerts, 1)
// Firing alerts should have precedence over normal alerts
require.Equal(t, "Alerting", rule.Alerts[0].State)
})
})
}
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, *acmock.Mock, PrometheusSrv) {