Alerting: Fix rules API pagination when using in-memory filters (#114334)
This commit is contained in:
Alexander Akhmetov
@@ -23,6 +23,7 @@ import (
|
||||
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
|
||||
"github.com/grafana/grafana/pkg/services/datasources"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/folder"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/accesscontrol"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
||||
@@ -327,6 +328,7 @@ func withLabels(labels data.Labels) forEachState {
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:gocyclo
|
||||
func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
timeNow = func() time.Time { return time.Date(2022, 3, 10, 14, 0, 0, 0, time.UTC) }
|
||||
orgID := int64(1)
|
||||
@@ -2219,6 +2221,575 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
require.True(t, foundNoProv, "should find rule without provenance")
|
||||
require.True(t, foundWithProv, "should find rule with provenance")
|
||||
})
|
||||
|
||||
t.Run("filter-aware pagination", func(t *testing.T) {
|
||||
createRulesWithState := func(t *testing.T, store *fakes.RuleStore, aim *fakeAlertInstanceManager,
|
||||
orgID int64, numGroups int, rulesPerGroup int,
|
||||
stateFunc func(groupIdx int) eval.State,
|
||||
healthFunc func(groupIdx int) error,
|
||||
stateMutators ...func(groupIdx int, s *state.State) *state.State) {
|
||||
t.Helper()
|
||||
|
||||
// create folders
|
||||
for i := 1; i <= numGroups; i++ {
|
||||
store.Folders[orgID] = append(store.Folders[orgID], &folder.Folder{
|
||||
ID: int64(i),
|
||||
UID: fmt.Sprintf("ns-%d", i),
|
||||
Title: fmt.Sprintf("Namespace %d", i),
|
||||
Fullpath: fmt.Sprintf("/namespace-%d", i),
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < numGroups; i++ {
|
||||
for j := 0; j < rulesPerGroup; j++ {
|
||||
rule := gen.With(gen.WithOrgID(orgID), func(r *ngmodels.AlertRule) {
|
||||
r.NamespaceUID = fmt.Sprintf("ns-%d", i+1)
|
||||
r.RuleGroup = fmt.Sprintf("group-%d", i+1)
|
||||
r.UID = fmt.Sprintf("rule-%d-%d", i+1, j+1)
|
||||
}, withClassicConditionSingleQuery()).GenerateRef()
|
||||
|
||||
alertState := stateFunc(i)
|
||||
healthErr := healthFunc(i)
|
||||
|
||||
aim.GenerateAlertInstances(orgID, rule.UID, 1, func(s *state.State) *state.State {
|
||||
s.State = alertState
|
||||
s.Error = healthErr
|
||||
s.Labels = data.Labels{"test": "label"}
|
||||
|
||||
for _, mutator := range stateMutators {
|
||||
s = mutator(i, s)
|
||||
}
|
||||
return s
|
||||
})
|
||||
store.PutRule(context.Background(), rule)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("state filter fetches multiple pages to fill group_limit", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 10 groups (2 rules each = 20 rules total): groups 1,3,5,7,9 firing, groups 2,4,6,8,10 normal
|
||||
// Request group_limit=3 with state=firing should fetch pages until 3 firing groups collected
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 10, 2,
|
||||
func(i int) eval.State {
|
||||
if i%2 == 0 {
|
||||
return eval.Alerting
|
||||
}
|
||||
return eval.Normal
|
||||
},
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request 3 groups with state=firing filter
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&group_limit=3", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 3 firing groups
|
||||
require.Len(t, res.Data.RuleGroups, 3)
|
||||
require.Equal(t, "group-1", res.Data.RuleGroups[0].Name)
|
||||
require.Equal(t, "group-3", res.Data.RuleGroups[1].Name)
|
||||
require.Equal(t, "group-5", res.Data.RuleGroups[2].Name)
|
||||
|
||||
// Verify all have firing alerts
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
hasFiring := false
|
||||
for _, rule := range rg.Rules {
|
||||
for _, alert := range rule.Alerts {
|
||||
if alert.State == eval.Alerting.String() {
|
||||
hasFiring = true
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, hasFiring)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("state filter continues when first page has no matches", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 8 groups (2 rules each = 16 rules total): first 4 normal, last 4 firing
|
||||
// Request state=firing with group_limit=2 should skip first 4 and return groups 5,6
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 8, 2,
|
||||
func(i int) eval.State {
|
||||
if i < 4 {
|
||||
return eval.Normal // groups 1-4 normal
|
||||
}
|
||||
return eval.Alerting // groups 5-8 firing
|
||||
},
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request 2 firing groups - should skip past the first page of normal rules
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&group_limit=2", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 2 firing groups
|
||||
require.Len(t, res.Data.RuleGroups, 2)
|
||||
require.Equal(t, "group-5", res.Data.RuleGroups[0].Name)
|
||||
require.Equal(t, "group-6", res.Data.RuleGroups[1].Name)
|
||||
})
|
||||
|
||||
t.Run("health filter fetches multiple pages", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 8 groups (2 rules each = 16 rules total): groups 1,3,5,7 with error health, groups 2,4,6,8 with ok health
|
||||
// Request health=error with group_limit=3 should fetch pages until 3 error groups collected
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 8, 2,
|
||||
func(i int) eval.State { return eval.Normal },
|
||||
func(i int) error {
|
||||
if i%2 == 0 {
|
||||
return fmt.Errorf("evaluation error")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Request 3 groups with health=error filter
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?health=error&group_limit=3", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 3 error groups
|
||||
require.Len(t, res.Data.RuleGroups, 3)
|
||||
require.Equal(t, "group-1", res.Data.RuleGroups[0].Name)
|
||||
require.Equal(t, "group-3", res.Data.RuleGroups[1].Name)
|
||||
require.Equal(t, "group-5", res.Data.RuleGroups[2].Name)
|
||||
|
||||
// Verify all have error health
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "error", rule.Health)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("combined state and health filters", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 10 groups (2 rules each = 20 rules total)
|
||||
// Groups 1-5: firing, groups 6-10: normal
|
||||
// Groups 1,3,5,7,9: ok health, groups 2,4,6,8,10: error health
|
||||
// Groups matching both filters (firing + ok): 1,3,5
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 10, 2,
|
||||
func(i int) eval.State {
|
||||
if i < 5 {
|
||||
return eval.Alerting
|
||||
}
|
||||
return eval.Normal
|
||||
},
|
||||
func(i int) error {
|
||||
if i%2 == 1 {
|
||||
return fmt.Errorf("evaluation error")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Request 3 groups with state=firing AND health=ok
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&health=ok&group_limit=3", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 3 groups matching both filters
|
||||
require.Len(t, res.Data.RuleGroups, 3)
|
||||
require.Equal(t, "group-1", res.Data.RuleGroups[0].Name)
|
||||
require.Equal(t, "group-3", res.Data.RuleGroups[1].Name)
|
||||
require.Equal(t, "group-5", res.Data.RuleGroups[2].Name)
|
||||
|
||||
// Verify all match both criteria
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "ok", rule.Health)
|
||||
hasFiring := false
|
||||
for _, alert := range rule.Alerts {
|
||||
if alert.State == eval.Alerting.String() {
|
||||
hasFiring = true
|
||||
}
|
||||
}
|
||||
require.True(t, hasFiring)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rule_limit hit before group_limit", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 5 groups (3 rules each = 15 rules total)
|
||||
// Request: group_limit=10, rule_limit=8
|
||||
// Expected: Should return groups 1-3 (9 rules total, exceeds limit but complete group included)
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 5, 3,
|
||||
func(i int) eval.State { return eval.Alerting }, // all firing
|
||||
func(i int) error { return nil }) // all healthy
|
||||
|
||||
// Request group_limit=10, rule_limit=8 - rule limit should hit first
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?group_limit=10&rule_limit=8", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Expected behavior with rule_limit=8:
|
||||
// Group 1: 3 rules (total: 3, under 8, continue)
|
||||
// Group 2: 3 rules (total: 6, under 8, continue)
|
||||
// Group 3: 3 rules (total: 9, exceeds 8 but we include complete group)
|
||||
// Result: 3 groups, 9 rules total
|
||||
totalRules := 0
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
t.Logf("Group %s has %d rules", rg.Name, len(rg.Rules))
|
||||
totalRules += len(rg.Rules)
|
||||
}
|
||||
t.Logf("Total: %d rules, %d groups", totalRules, len(res.Data.RuleGroups))
|
||||
require.Equal(t, 9, totalRules)
|
||||
require.Equal(t, 3, len(res.Data.RuleGroups))
|
||||
})
|
||||
|
||||
t.Run("rule_limit without group_limit", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 10 groups (2 rules each = 20 rules total)
|
||||
// Request rule_limit=7 should stop after 4 groups (8 rules total)
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 10, 2,
|
||||
func(i int) eval.State { return eval.Alerting },
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request rule_limit=7 only (group_limit unlimited)
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?rule_limit=7", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 4 groups (8 rules total, stops after exceeding 7)
|
||||
totalRules := 0
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
totalRules += len(rg.Rules)
|
||||
}
|
||||
require.Equal(t, 8, totalRules)
|
||||
require.Equal(t, 4, len(res.Data.RuleGroups))
|
||||
require.NotEmpty(t, res.Data.NextToken)
|
||||
})
|
||||
|
||||
t.Run("empty page in middle of pagination", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 15 groups (2 rules each = 30 rules total): groups 1-3 firing, 4-8 normal, 9-13 firing, 14-15 normal
|
||||
// Request state=firing with group_limit=5 should skip over normal groups
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 15, 2,
|
||||
func(i int) eval.State {
|
||||
groupNum := i + 1
|
||||
if groupNum <= 3 || (groupNum >= 9 && groupNum <= 13) {
|
||||
return eval.Alerting
|
||||
}
|
||||
return eval.Normal
|
||||
},
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request state=firing, group_limit=5
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&group_limit=5", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Should return 5 firing groups from pages 1 and 3 (skipping empty page 2)
|
||||
require.Len(t, res.Data.RuleGroups, 5)
|
||||
|
||||
// Verify all are firing
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
for _, alert := range rule.Alerts {
|
||||
require.Equal(t, eval.Alerting.String(), alert.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("group_limit=0 returns empty response", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 1 group (2 rules) to verify group_limit=0 returns empty
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 1, 2,
|
||||
func(i int) eval.State { return eval.Alerting },
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request group_limit=0
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?group_limit=0", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
require.Len(t, res.Data.RuleGroups, 0)
|
||||
require.Empty(t, res.Data.NextToken)
|
||||
})
|
||||
|
||||
t.Run("resume with token and filters active", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 10 groups (2 rules each = 20 rules total): odd groups firing, even groups normal
|
||||
// Test pagination continuation with state filter: first page returns groups 1,3 then second page returns groups 5,7
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 10, 2,
|
||||
func(i int) eval.State {
|
||||
if i%2 == 0 {
|
||||
return eval.Alerting
|
||||
}
|
||||
return eval.Normal
|
||||
},
|
||||
func(i int) error { return nil })
|
||||
|
||||
// First request: state=firing, group_limit=2
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&group_limit=2", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res1 apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res1))
|
||||
|
||||
// Should return 2 firing groups (1, 3)
|
||||
require.Len(t, res1.Data.RuleGroups, 2)
|
||||
require.NotEmpty(t, res1.Data.NextToken)
|
||||
|
||||
// Verify both are firing
|
||||
for _, rg := range res1.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "firing", rule.State)
|
||||
}
|
||||
}
|
||||
|
||||
// Second request: resume with token AND filter still active
|
||||
req2, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/rules?state=firing&group_limit=2&group_next_token=%s", res1.Data.NextToken), nil)
|
||||
require.NoError(t, err)
|
||||
c2 := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req2},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp2 := api.RouteGetRuleStatuses(c2)
|
||||
require.Equal(t, http.StatusOK, resp2.Status())
|
||||
|
||||
var res2 apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp2.Body(), &res2))
|
||||
|
||||
// Should return next 2 firing groups (5, 7)
|
||||
require.Len(t, res2.Data.RuleGroups, 2)
|
||||
require.NotEmpty(t, res2.Data.NextToken)
|
||||
|
||||
// Verify both are firing
|
||||
for _, rg := range res2.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "firing", rule.State)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify we got different groups than first request
|
||||
firstGroupNames := make(map[string]bool)
|
||||
for _, rg := range res1.Data.RuleGroups {
|
||||
firstGroupNames[rg.Name] = true
|
||||
}
|
||||
for _, rg := range res2.Data.RuleGroups {
|
||||
require.False(t, firstGroupNames[rg.Name])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rule_limit with state filter", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 10 groups (2 rules each = 20 rules total), alternating firing/normal
|
||||
// Firing groups: 1,3,5,7,9 (5 groups × 2 rules = 10 firing rules)
|
||||
// Request state=firing with rule_limit=7 should return groups 1,3,5,7 (8 rules)
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 10, 2,
|
||||
func(i int) eval.State {
|
||||
if i%2 == 0 {
|
||||
return eval.Alerting
|
||||
}
|
||||
return eval.Normal
|
||||
},
|
||||
func(i int) error { return nil })
|
||||
|
||||
// Request state=firing, rule_limit=7
|
||||
// Should fetch multiple pages to accumulate 7+ firing rules
|
||||
// Groups 1, 3, 5 = 6 rules (under limit)
|
||||
// Group 7 = +2 rules = 8 total (exceeds 7, but we include full group)
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&rule_limit=7", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Count total firing rules returned
|
||||
totalRules := 0
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "firing", rule.State)
|
||||
totalRules++
|
||||
}
|
||||
}
|
||||
|
||||
// Should return 4 firing groups (1,3,5,7) with 8 rules total
|
||||
require.Equal(t, 8, totalRules)
|
||||
require.Equal(t, 4, len(res.Data.RuleGroups))
|
||||
require.NotEmpty(t, res.Data.NextToken)
|
||||
})
|
||||
|
||||
t.Run("rule_limit with health filter", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, api := setupAPI(t)
|
||||
|
||||
// Create 8 groups (3 rules each = 24 rules total), alternating ok/error health
|
||||
// Error groups: 1,3,5,7 (4 groups × 3 rules = 12 error rules)
|
||||
// Request health=error with rule_limit=8 should return groups 1,3,5 (9 rules)
|
||||
createRulesWithState(t, fakeStore, fakeAIM, orgID, 8, 3,
|
||||
func(i int) eval.State { return eval.Normal },
|
||||
func(i int) error {
|
||||
if i%2 == 0 {
|
||||
return fmt.Errorf("evaluation error")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Request health=error, rule_limit=8
|
||||
// Should fetch multiple pages to accumulate 8+ error rules
|
||||
// Groups 1, 3 = 6 rules (under limit)
|
||||
// Group 5 = +3 rules = 9 total (exceeds 8, but we include full group)
|
||||
req, err := http.NewRequest("GET", "/api/v1/rules?health=error&rule_limit=8", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: req},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
Permissions: queryPermissions,
|
||||
},
|
||||
}
|
||||
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Count total error rules returned
|
||||
totalRules := 0
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
for _, rule := range rg.Rules {
|
||||
require.Equal(t, "error", rule.Health)
|
||||
totalRules++
|
||||
}
|
||||
}
|
||||
|
||||
// Should return 3 error groups (1,3,5) with 9 rules total
|
||||
require.Equal(t, 9, totalRules)
|
||||
require.Equal(t, 3, len(res.Data.RuleGroups))
|
||||
require.NotEmpty(t, res.Data.NextToken)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv) {
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type RuleStoreReader interface {
|
||||
@@ -429,6 +430,177 @@ func RuleAlertStateMutatorGenerator(manager state.AlertInstanceManager) RuleAler
|
||||
}
|
||||
}
|
||||
|
||||
// paginationContext holds limits and filters for filter-aware pagination
|
||||
type paginationContext struct {
|
||||
opts RuleGroupStatusesOptions
|
||||
provenanceRecords map[string]ngmodels.Provenance
|
||||
ruleStatusMutator RuleStatusMutator
|
||||
alertStateMutator RuleAlertStateMutator
|
||||
|
||||
// Query parameters
|
||||
namespaceUIDs []string
|
||||
ruleUIDs []string
|
||||
dashboardUID string
|
||||
panelID int64
|
||||
ruleGroups []string
|
||||
receiverName string
|
||||
title string
|
||||
searchRuleGroup string
|
||||
ruleType ngmodels.RuleTypeFilter
|
||||
ruleNamesSet map[string]struct{}
|
||||
|
||||
// Filters
|
||||
stateFilterSet map[eval.State]struct{}
|
||||
healthFilterSet map[string]struct{}
|
||||
matchers labels.Matchers
|
||||
labelOptions []ngmodels.LabelOption
|
||||
limitAlertsPerRule int64
|
||||
limitRulesPerGroup int64
|
||||
}
|
||||
|
||||
// pageResult is the result of fetching and filtering of one page
|
||||
type pageResult struct {
|
||||
groups []apimodels.RuleGroup
|
||||
totalsDelta map[string]int64
|
||||
nextToken string
|
||||
hasMore bool
|
||||
}
|
||||
|
||||
func accumulateTotals(dest, source map[string]int64) {
|
||||
for k, v := range source {
|
||||
dest[k] += v
|
||||
}
|
||||
}
|
||||
|
||||
// fetchAndFilterPage fetches one page from the store and applies filters
|
||||
func (ctx *paginationContext) fetchAndFilterPage(log log.Logger, store ListAlertRulesStoreV2, span trace.Span, token string, remainingGroups, remainingRules int64) (pageResult, error) {
|
||||
byGroupQuery := ngmodels.ListAlertRulesExtendedQuery{
|
||||
ListAlertRulesQuery: ngmodels.ListAlertRulesQuery{
|
||||
OrgID: ctx.opts.OrgID,
|
||||
NamespaceUIDs: ctx.namespaceUIDs,
|
||||
RuleUIDs: ctx.ruleUIDs,
|
||||
DashboardUID: ctx.dashboardUID,
|
||||
PanelID: ctx.panelID,
|
||||
RuleGroups: ctx.ruleGroups,
|
||||
ReceiverName: ctx.receiverName,
|
||||
SearchTitle: ctx.title,
|
||||
SearchRuleGroup: ctx.searchRuleGroup,
|
||||
},
|
||||
RuleType: ctx.ruleType,
|
||||
Limit: remainingGroups,
|
||||
RuleLimit: remainingRules,
|
||||
ContinueToken: token,
|
||||
}
|
||||
|
||||
ruleList, newToken, err := store.ListAlertRulesByGroup(ctx.opts.Ctx, &byGroupQuery)
|
||||
if err != nil {
|
||||
return pageResult{}, err
|
||||
}
|
||||
|
||||
span.SetAttributes(
|
||||
attribute.Int("store_rule_list_len", len(ruleList)),
|
||||
attribute.Bool("store_continue_token_set", newToken != ""),
|
||||
)
|
||||
span.AddEvent("Alert rules retrieved from store")
|
||||
|
||||
groupedRules := getGroupedRules(log, ruleList, ctx.ruleNamesSet, ctx.opts.AllowedNamespaces)
|
||||
|
||||
result := pageResult{
|
||||
groups: make([]apimodels.RuleGroup, 0, len(groupedRules)),
|
||||
totalsDelta: make(map[string]int64),
|
||||
nextToken: newToken,
|
||||
hasMore: newToken != "",
|
||||
}
|
||||
|
||||
for _, rg := range groupedRules {
|
||||
ruleGroup, totals := toRuleGroup(
|
||||
log, rg.GroupKey, rg.Folder, rg.Rules,
|
||||
ctx.provenanceRecords, ctx.limitAlertsPerRule,
|
||||
ctx.stateFilterSet, ctx.matchers, ctx.labelOptions,
|
||||
ctx.ruleStatusMutator, ctx.alertStateMutator,
|
||||
)
|
||||
ruleGroup.Totals = totals
|
||||
accumulateTotals(result.totalsDelta, totals)
|
||||
|
||||
if len(ctx.stateFilterSet) > 0 {
|
||||
filterRulesByState(ruleGroup, ctx.stateFilterSet)
|
||||
}
|
||||
|
||||
if len(ctx.healthFilterSet) > 0 {
|
||||
filterRulesByHealth(ruleGroup, ctx.healthFilterSet)
|
||||
}
|
||||
|
||||
if ctx.limitRulesPerGroup > -1 && int64(len(ruleGroup.Rules)) > ctx.limitRulesPerGroup {
|
||||
ruleGroup.Rules = ruleGroup.Rules[0:ctx.limitRulesPerGroup]
|
||||
}
|
||||
|
||||
if len(ruleGroup.Rules) > 0 {
|
||||
result.groups = append(result.groups, *ruleGroup)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// paginateRuleGroups fetches pages until limits are satisfied applying filters at each step
|
||||
func paginateRuleGroups(log log.Logger, store ListAlertRulesStoreV2, ctx *paginationContext, span trace.Span, maxGroups, maxRules int64, startToken string) ([]apimodels.RuleGroup, map[string]int64, string, error) {
|
||||
allGroups := []apimodels.RuleGroup{}
|
||||
rulesTotals := make(map[string]int64)
|
||||
|
||||
continueToken := startToken
|
||||
groupsReturned := int64(0)
|
||||
rulesReturned := int64(0)
|
||||
|
||||
for {
|
||||
remainingGroups := maxGroups
|
||||
if maxGroups > 0 {
|
||||
remainingGroups = maxGroups - groupsReturned
|
||||
if remainingGroups <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
remainingRules := maxRules
|
||||
if maxRules > 0 {
|
||||
remainingRules = maxRules - rulesReturned
|
||||
if remainingRules <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
page, err := ctx.fetchAndFilterPage(log, store, span, continueToken, remainingGroups, remainingRules)
|
||||
if err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
|
||||
accumulateTotals(rulesTotals, page.totalsDelta)
|
||||
|
||||
// Add groups and check limits
|
||||
for _, group := range page.groups {
|
||||
allGroups = append(allGroups, group)
|
||||
groupsReturned++
|
||||
rulesReturned += int64(len(group.Rules))
|
||||
|
||||
// Check if we've hit limits
|
||||
if (maxGroups > 0 && groupsReturned == maxGroups) || (maxRules > 0 && rulesReturned >= maxRules) {
|
||||
return allGroups, rulesTotals, page.nextToken, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !page.hasMore {
|
||||
return allGroups, rulesTotals, "", nil
|
||||
}
|
||||
|
||||
if page.nextToken == continueToken {
|
||||
log.Warn("Pagination loop detected same token, stopping", "token", page.nextToken)
|
||||
return allGroups, rulesTotals, page.nextToken, nil
|
||||
}
|
||||
|
||||
continueToken = page.nextToken
|
||||
}
|
||||
|
||||
return allGroups, rulesTotals, continueToken, nil
|
||||
}
|
||||
|
||||
func PrepareRuleGroupStatusesV2(log log.Logger, store ListAlertRulesStoreV2, opts RuleGroupStatusesOptions, ruleStatusMutator RuleStatusMutator, alertStateMutator RuleAlertStateMutator, provenanceRecords map[string]ngmodels.Provenance) apimodels.RuleResponse {
|
||||
ctx, span := tracer.Start(opts.Ctx, "api.prometheus.PrepareRuleGroupStatusesV2")
|
||||
defer span.End()
|
||||
@@ -591,62 +763,38 @@ func PrepareRuleGroupStatusesV2(log log.Logger, store ListAlertRulesStoreV2, opt
|
||||
}
|
||||
span.SetAttributes(attribute.Int("rule_name_count", len(ruleNamesSet)))
|
||||
|
||||
byGroupQuery := ngmodels.ListAlertRulesExtendedQuery{
|
||||
ListAlertRulesQuery: ngmodels.ListAlertRulesQuery{
|
||||
OrgID: opts.OrgID,
|
||||
NamespaceUIDs: namespaceUIDs,
|
||||
RuleUIDs: ruleUIDs,
|
||||
DashboardUID: dashboardUID,
|
||||
PanelID: panelID,
|
||||
RuleGroups: ruleGroups,
|
||||
ReceiverName: receiverName,
|
||||
SearchTitle: title,
|
||||
SearchRuleGroup: searchRuleGroup,
|
||||
},
|
||||
RuleType: ruleType,
|
||||
Limit: maxGroups,
|
||||
RuleLimit: maxRules,
|
||||
ContinueToken: nextToken,
|
||||
pagCtx := &paginationContext{
|
||||
opts: opts,
|
||||
provenanceRecords: provenanceRecords,
|
||||
ruleStatusMutator: ruleStatusMutator,
|
||||
alertStateMutator: alertStateMutator,
|
||||
namespaceUIDs: namespaceUIDs,
|
||||
ruleUIDs: ruleUIDs,
|
||||
dashboardUID: dashboardUID,
|
||||
panelID: panelID,
|
||||
ruleGroups: ruleGroups,
|
||||
receiverName: receiverName,
|
||||
title: title,
|
||||
searchRuleGroup: searchRuleGroup,
|
||||
ruleType: ruleType,
|
||||
ruleNamesSet: ruleNamesSet,
|
||||
stateFilterSet: stateFilterSet,
|
||||
healthFilterSet: healthFilterSet,
|
||||
matchers: matchers,
|
||||
labelOptions: labelOptions,
|
||||
limitAlertsPerRule: limitAlertsPerRule,
|
||||
limitRulesPerGroup: limitRulesPerGroup,
|
||||
}
|
||||
ruleList, continueToken, err := store.ListAlertRulesByGroup(opts.Ctx, &byGroupQuery)
|
||||
|
||||
groups, rulesTotals, continueToken, err := paginateRuleGroups(log, store, pagCtx, span, maxGroups, maxRules, nextToken)
|
||||
if err != nil {
|
||||
ruleResponse.Status = "error"
|
||||
ruleResponse.Error = fmt.Sprintf("failure getting rules: %s", err.Error())
|
||||
ruleResponse.ErrorType = apiv1.ErrServer
|
||||
return ruleResponse
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.Int("store_rule_list_len", len(ruleList)),
|
||||
attribute.Bool("store_continue_token_set", continueToken != ""),
|
||||
)
|
||||
span.AddEvent("Alert rules retrieved from store")
|
||||
|
||||
groupedRules := getGroupedRules(log, ruleList, ruleNamesSet, opts.AllowedNamespaces)
|
||||
rulesTotals := make(map[string]int64, len(groupedRules))
|
||||
for _, rg := range groupedRules {
|
||||
ruleGroup, totals := toRuleGroup(log, rg.GroupKey, rg.Folder, rg.Rules, provenanceRecords, limitAlertsPerRule, stateFilterSet, matchers, labelOptions, ruleStatusMutator, alertStateMutator)
|
||||
ruleGroup.Totals = totals
|
||||
for k, v := range totals {
|
||||
rulesTotals[k] += v
|
||||
}
|
||||
|
||||
if len(stateFilterSet) > 0 {
|
||||
filterRulesByState(ruleGroup, stateFilterSet)
|
||||
}
|
||||
|
||||
if len(healthFilterSet) > 0 {
|
||||
filterRulesByHealth(ruleGroup, healthFilterSet)
|
||||
}
|
||||
|
||||
if limitRulesPerGroup > -1 && int64(len(ruleGroup.Rules)) > limitRulesPerGroup {
|
||||
ruleGroup.Rules = ruleGroup.Rules[0:limitRulesPerGroup]
|
||||
}
|
||||
|
||||
if len(ruleGroup.Rules) > 0 {
|
||||
ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
|
||||
}
|
||||
}
|
||||
|
||||
ruleResponse.Data.RuleGroups = groups
|
||||
ruleResponse.Data.NextToken = continueToken
|
||||
|
||||
// Only return Totals if there is no pagination
|
||||
@@ -981,6 +1129,7 @@ func toRuleGroup(log log.Logger, groupKey ngmodels.AlertRuleGroupKey, folderFull
|
||||
|
||||
// mutate rule for alert states
|
||||
totals, totalsFiltered := ruleAlertStateMutator(rule, &alertingRule, stateFilterSet, matchers, labelOptions)
|
||||
|
||||
if alertingRule.State != "" {
|
||||
rulesTotals[alertingRule.State] += 1
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user