public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Fetch alert rule provenances for a page of rules only (#115643)

Browse Source 
* Alerting: Fetch alert rule provenances for a page of rules only

* error when failed to fetch provenance
This commit is contained in:
Alexander Akhmetov
2025-12-24 13:41:46 +01:00
committed by GitHub
parent c38e515dec
commit e38f007d30
8 changed files with 331 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_prometheus_test.go
+134
View File
@@ -2369,6 +2369,140 @@ func TestRouteGetRuleStatuses(t *testing.T) {
}
})
t.Run("multi-page pagination loads provenance correctly", func(t *testing.T) {
fakeStore, fakeAIM, api, fakeProvisioning := setupAPIFull(t)
// Create 3 groups with 1 rule each: groups 1 and 3 firing, group 2 normal
for i := 1; i <= 3; i++ {
rule := gen.With(gen.WithOrgID(orgID), func(r *ngmodels.AlertRule) {
r.NamespaceUID = "ns-1"
r.RuleGroup = fmt.Sprintf("group-%d", i)
r.UID = fmt.Sprintf("rule-%d", i)
}, withClassicConditionSingleQuery()).GenerateRef()
alertState := eval.Normal
if i != 2 {
alertState = eval.Alerting
}
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, func(s *state.State) *state.State {
s.State = alertState
s.Labels = data.Labels{"test": "label"}
return s
})
fakeStore.PutRule(context.Background(), rule)
}
// Set provenance for all rules
err := fakeProvisioning.SetProvenance(context.Background(),
&ngmodels.AlertRule{UID: "rule-1", OrgID: orgID}, orgID, ngmodels.ProvenanceAPI)
require.NoError(t, err)
err = fakeProvisioning.SetProvenance(context.Background(),
&ngmodels.AlertRule{UID: "rule-3", OrgID: orgID}, orgID, ngmodels.ProvenanceFile)
require.NoError(t, err)
// Request firing groups with group_limit=2 - fetches multiple pages, skipping group 2
req, err := http.NewRequest("GET", "/api/v1/rules?state=firing&group_limit=2", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: req},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusOK, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
// Should return 2 firing groups
require.Len(t, res.Data.RuleGroups, 2)
require.Equal(t, "group-1", res.Data.RuleGroups[0].Name)
require.Equal(t, apimodels.Provenance(ngmodels.ProvenanceAPI), res.Data.RuleGroups[0].Rules[0].Provenance)
require.Equal(t, "group-3", res.Data.RuleGroups[1].Name)
require.Equal(t, apimodels.Provenance(ngmodels.ProvenanceFile), res.Data.RuleGroups[1].Rules[0].Provenance)
})
t.Run("provenance fetch error returns error response in paginated mode", func(t *testing.T) {
fakeStore, fakeAIM, api, fakeProvisioning := setupAPIFull(t)
rule := gen.With(gen.WithOrgID(orgID), func(r *ngmodels.AlertRule) {
r.NamespaceUID = "ns-1"
r.RuleGroup = "group-1"
r.UID = "rule-1"
}, withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, func(s *state.State) *state.State {
s.State = eval.Alerting
s.Labels = data.Labels{"test": "label"}
return s
})
fakeStore.PutRule(context.Background(), rule)
fakeProvisioning.GetProvenancesByUIDsFunc = func(ctx context.Context, orgID int64, resourceType string, uids []string) (map[string]ngmodels.Provenance, error) {
return nil, errors.New("database connection failed")
}
req, err := http.NewRequest("GET", "/api/v1/rules?group_limit=10", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: req},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusInternalServerError, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Equal(t, "error", res.Status)
require.Contains(t, res.Error, "failed to load provenance")
})
t.Run("provenance fetch error returns error response in non-paginated mode", func(t *testing.T) {
fakeStore, fakeAIM, api, fakeProvisioning := setupAPIFull(t)
rule := gen.With(gen.WithOrgID(orgID), func(r *ngmodels.AlertRule) {
r.NamespaceUID = "ns-1"
r.RuleGroup = "group-1"
r.UID = "rule-1"
}, withClassicConditionSingleQuery()).GenerateRef()
fakeAIM.GenerateAlertInstances(orgID, rule.UID, 1, func(s *state.State) *state.State {
s.State = eval.Alerting
s.Labels = data.Labels{"test": "label"}
return s
})
fakeStore.PutRule(context.Background(), rule)
fakeProvisioning.GetProvenancesFunc = func(ctx context.Context, orgID int64, resourceType string) (map[string]ngmodels.Provenance, error) {
return nil, errors.New("database connection failed")
}
req, err := http.NewRequest("GET", "/api/v1/rules", nil)
require.NoError(t, err)
c := &contextmodel.ReqContext{
Context: &web.Context{Req: req},
SignedInUser: &user.SignedInUser{
OrgID: orgID,
Permissions: queryPermissions,
},
}
resp := api.RouteGetRuleStatuses(c)
require.Equal(t, http.StatusInternalServerError, resp.Status())
var res apimodels.RuleResponse
require.NoError(t, json.Unmarshal(resp.Body(), &res))
require.Equal(t, "error", res.Status)
require.Contains(t, res.Error, "failed to load provenance")
})
t.Run("state filter continues when first page has no matches", func(t *testing.T) {
fakeStore, fakeAIM, api := setupAPI(t)
pkg/services/ngalert/api/prometheus/api_prometheus.go
+37 -11
View File
@@ -54,6 +54,7 @@ type StatusReader interface {
type ProvenanceStore interface {
GetProvenances(ctx context.Context, org int64, resourceType string) (map[string]ngmodels.Provenance, error)
GetProvenancesByUIDs(ctx context.Context, org int64, resourceType string, uids []string) (map[string]ngmodels.Provenance, error)
}
type PrometheusSrv struct {
@@ -328,14 +329,6 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
span.AddEvent("User permissions checked")
span.SetAttributes(attribute.Int("allowedNamespaces", len(allowedNamespaces)))
provenanceRecords, err := srv.provenanceStore.GetProvenances(c.Req.Context(), c.GetOrgID(), (&ngmodels.AlertRule{}).ResourceType())
if err != nil {
ruleResponse.Status = "error"
ruleResponse.Error = fmt.Sprintf("failed to get provenances visible to the user: %s", err.Error())
ruleResponse.ErrorType = apiv1.ErrServer
return response.JSON(ruleResponse.HTTPStatusCode(), ruleResponse)
}
ruleResponse = PrepareRuleGroupStatusesV2(
srv.log,
srv.store,
@@ -347,7 +340,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
},
RuleStatusMutatorGenerator(srv.status),
RuleAlertStateMutatorGenerator(srv.manager),
provenanceRecords,
srv.provenanceStore,
)
return response.JSON(ruleResponse.HTTPStatusCode(), ruleResponse)
@@ -454,6 +447,7 @@ func RuleAlertStateMutatorGenerator(manager state.AlertInstanceManager) RuleAler
type paginationContext struct {
opts RuleGroupStatusesOptions
provenanceRecords map[string]ngmodels.Provenance
provenanceStore ProvenanceStore
ruleStatusMutator RuleStatusMutator
alertStateMutator RuleAlertStateMutator
@@ -532,6 +526,37 @@ func (ctx *paginationContext) fetchAndFilterPage(log log.Logger, store ListAlert
)
span.AddEvent("Alert rules retrieved from store")
// Load provenance for this page's rules
if ctx.provenanceStore != nil {
maxGroups := getInt64WithDefault(ctx.opts.Query, "group_limit", -1)
maxRules := getInt64WithDefault(ctx.opts.Query, "rule_limit", -1)
if maxGroups > 0 || maxRules > 0 {
// Paginated, fetch and merge provenances for this page
uids := make([]string, 0, len(ruleList))
for _, rule := range ruleList {
uids = append(uids, rule.UID)
}
pageProvenances, err := ctx.provenanceStore.GetProvenancesByUIDs(ctx.opts.Ctx, ctx.opts.OrgID, (&ngmodels.AlertRule{}).ResourceType(), uids)
if err != nil {
return pageResult{}, fmt.Errorf("failed to load provenance: %w", err)
}
if ctx.provenanceRecords == nil {
ctx.provenanceRecords = pageProvenances
} else {
maps.Copy(ctx.provenanceRecords, pageProvenances)
}
} else if ctx.provenanceRecords == nil {
// Not paginated, fetch all once
var err error
ctx.provenanceRecords, err = ctx.provenanceStore.GetProvenances(ctx.opts.Ctx, ctx.opts.OrgID, (&ngmodels.AlertRule{}).ResourceType())
if err != nil {
return pageResult{}, fmt.Errorf("failed to load provenance: %w", err)
}
}
}
span.AddEvent("Provenances retrieved from store")
groupedRules := getGroupedRules(log, ruleList, ctx.ruleNamesSet, ctx.opts.AllowedNamespaces)
result := pageResult{
@@ -643,7 +668,7 @@ func paginateRuleGroups(log log.Logger, store ListAlertRulesStoreV2, ctx *pagina
return allGroups, rulesTotals, continueToken, nil
}
func PrepareRuleGroupStatusesV2(log log.Logger, store ListAlertRulesStoreV2, opts RuleGroupStatusesOptions, ruleStatusMutator RuleStatusMutator, alertStateMutator RuleAlertStateMutator, provenanceRecords map[string]ngmodels.Provenance) apimodels.RuleResponse {
func PrepareRuleGroupStatusesV2(log log.Logger, store ListAlertRulesStoreV2, opts RuleGroupStatusesOptions, ruleStatusMutator RuleStatusMutator, alertStateMutator RuleAlertStateMutator, provenanceStore ProvenanceStore) apimodels.RuleResponse {
ctx, span := tracer.Start(opts.Ctx, "api.prometheus.PrepareRuleGroupStatusesV2")
defer span.End()
opts.Ctx = ctx
@@ -835,7 +860,8 @@ func PrepareRuleGroupStatusesV2(log log.Logger, store ListAlertRulesStoreV2, opt
span.SetAttributes(attribute.Bool("compact", compact))
pagCtx := &paginationContext{
opts: opts,
provenanceRecords: provenanceRecords,
provenanceRecords: nil,
provenanceStore: provenanceStore,
ruleStatusMutator: ruleStatusMutator,
alertStateMutator: alertStateMutator,
namespaceUIDs: namespaceUIDs,