From 74e800e427a4f0f0f84b551f4e0a7ee330921411 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Tue, 17 Jun 2025 17:31:42 -0400 Subject: [PATCH] Alerting: Add provenance to Prometheus API (#106596) This commit adds provenance information to the Prometheus API in the ngalert service to enable compatibility with the new alert list page. --- pkg/services/ngalert/api/api.go | 2 +- .../ngalert/api/api_prometheus_test.go | 67 +++++++++++++++- .../ngalert/api/prometheus/api_prometheus.go | 80 ++++++++++++------- .../ngalert/api/tooling/definitions/prom.go | 1 + 4 files changed, 121 insertions(+), 29 deletions(-) diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index 916bb62b8de..4b887adc4b7 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -120,7 +120,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) { api.RegisterPrometheusApiEndpoints(NewForkingProm( api.DatasourceCache, NewLotexProm(proxy, logger), - apiprometheus.NewPrometheusSrv(logger, api.StateManager, api.Scheduler, api.RuleStore, ruleAuthzService), + apiprometheus.NewPrometheusSrv(logger, api.StateManager, api.Scheduler, api.RuleStore, ruleAuthzService, api.ProvenanceStore), ), m) // Register endpoints for proxying to Cortex Ruler-compatible backends. api.RegisterRulerApiEndpoints(NewForkingRuler( diff --git a/pkg/services/ngalert/api/api_prometheus_test.go b/pkg/services/ngalert/api/api_prometheus_test.go index 796cbb50518..9755a28505c 100644 --- a/pkg/services/ngalert/api/api_prometheus_test.go +++ b/pkg/services/ngalert/api/api_prometheus_test.go @@ -749,6 +749,7 @@ func TestRouteGetRuleStatuses(t *testing.T) { fakeSch, ruleStore, &fakeRuleAccessControlService{}, + fakes.NewFakeProvisioningStore(), ) response := api.RouteGetRuleStatuses(c) @@ -811,6 +812,7 @@ func TestRouteGetRuleStatuses(t *testing.T) { newFakeSchedulerReader(t).setupStates(fakeAIM), ruleStore, accesscontrol.NewRuleService(acimpl.ProvideAccessControl(featuremgmt.WithFeatures())), + fakes.NewFakeProvisioningStore(), ) permissions := createPermissionsForRules(slices.Concat(rulesInGroup1, rulesInGroup2, rulesInGroup3), orgID) @@ -937,6 +939,7 @@ func TestRouteGetRuleStatuses(t *testing.T) { newFakeSchedulerReader(t).setupStates(fakeAIM), ruleStore, accesscontrol.NewRuleService(acimpl.ProvideAccessControl(featuremgmt.WithFeatures())), + fakes.NewFakeProvisioningStore(), ) permissions := createPermissionsForRules(allRules, orgID) @@ -1081,6 +1084,7 @@ func TestRouteGetRuleStatuses(t *testing.T) { newFakeSchedulerReader(t).setupStates(fakeAIM), ruleStore, accesscontrol.NewRuleService(acimpl.ProvideAccessControl(featuremgmt.WithFeatures())), + fakes.NewFakeProvisioningStore(), ) c := &contextmodel.ReqContext{Context: &web.Context{Req: req}, SignedInUser: &user.SignedInUser{OrgID: orgID, Permissions: createPermissionsForRules(rules, orgID)}} @@ -1980,13 +1984,73 @@ func TestRouteGetRuleStatuses(t *testing.T) { require.Equal(t, "webhook-a", rg.Rules[0].NotificationSettings.Receiver) }) }) + + t.Run("provenance as expected", func(t *testing.T) { + fakeStore, fakeAIM, api, provStore := setupAPIFull(t) + // Rule without provenance + ruleNoProv := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef() + fakeAIM.GenerateAlertInstances(orgID, ruleNoProv.UID, 1) + fakeStore.PutRule(context.Background(), ruleNoProv) + + // Rule with provenance + ruleWithProv := gen.With(gen.WithOrgID(orgID), asFixture(), withClassicConditionSingleQuery()).GenerateRef() + ruleWithProv.UID = "provRuleUID" + ruleWithProv.Title = "ProvisionedRule" + fakeAIM.GenerateAlertInstances(orgID, ruleWithProv.UID, 1) + fakeStore.PutRule(context.Background(), ruleWithProv) + + // Add provenance for ruleWithProv + err := provStore.SetProvenance(context.Background(), ruleWithProv, orgID, ngmodels.ProvenanceAPI) + require.NoError(t, err) + + req, err := http.NewRequest("GET", "/api/v1/rules", nil) + require.NoError(t, err) + c := &contextmodel.ReqContext{ + Context: &web.Context{Req: req}, + SignedInUser: &user.SignedInUser{ + OrgID: orgID, + Permissions: map[int64]map[string][]string{orgID: {datasources.ActionQuery: {datasources.ScopeAll}}}, + }, + } + + resp := api.RouteGetRuleStatuses(c) + require.Equal(t, http.StatusOK, resp.Status()) + var res apimodels.RuleResponse + require.NoError(t, json.Unmarshal(resp.Body(), &res)) + + // Should have two rules in one group + require.Len(t, res.Data.RuleGroups, 1) + rg := res.Data.RuleGroups[0] + require.Len(t, rg.Rules, 2) + + // Find rules by UID + var foundNoProv, foundWithProv bool + for _, rule := range rg.Rules { + switch rule.UID { + case ruleNoProv.UID: + foundNoProv = true + require.Equal(t, apimodels.Provenance(ngmodels.ProvenanceNone), rule.Provenance, "non-provisioned rule should have empty provenance") + case ruleWithProv.UID: + foundWithProv = true + require.Equal(t, apimodels.Provenance(ngmodels.ProvenanceAPI), rule.Provenance, "provisioned rule should have provenance set") + } + } + require.True(t, foundNoProv, "should find rule without provenance") + require.True(t, foundWithProv, "should find rule with provenance") + }) } func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv) { + fakeStore, fakeAIM, api, _ := setupAPIFull(t) + return fakeStore, fakeAIM, api +} + +func setupAPIFull(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv, *fakes.FakeProvisioningStore) { fakeStore := fakes.NewRuleStore(t) fakeAIM := NewFakeAlertInstanceManager(t) fakeSch := newFakeSchedulerReader(t).setupStates(fakeAIM) fakeAuthz := &fakeRuleAccessControlService{} + fakeProvisioning := fakes.NewFakeProvisioningStore() api := *NewPrometheusSrv( log.NewNopLogger(), @@ -1994,9 +2058,10 @@ func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, Promet fakeSch, fakeStore, fakeAuthz, + fakeProvisioning, ) - return fakeStore, fakeAIM, api + return fakeStore, fakeAIM, api, fakeProvisioning } func generateRuleAndInstanceWithQuery(t *testing.T, orgID int64, fakeAIM *fakeAlertInstanceManager, fakeStore *fakes.RuleStore, query ngmodels.AlertRuleMutator, additionalMutators ...ngmodels.AlertRuleMutator) { diff --git a/pkg/services/ngalert/api/prometheus/api_prometheus.go b/pkg/services/ngalert/api/prometheus/api_prometheus.go index d1208a8b4d6..2e753a6db1d 100644 --- a/pkg/services/ngalert/api/prometheus/api_prometheus.go +++ b/pkg/services/ngalert/api/prometheus/api_prometheus.go @@ -41,21 +41,27 @@ type StatusReader interface { Status(key ngmodels.AlertRuleKey) (ngmodels.RuleStatus, bool) } -type PrometheusSrv struct { - log log.Logger - manager state.AlertInstanceManager - status StatusReader - store RuleStoreReader - authz RuleGroupAccessControlService +type ProvenanceStore interface { + GetProvenances(ctx context.Context, org int64, resourceType string) (map[string]ngmodels.Provenance, error) } -func NewPrometheusSrv(log log.Logger, manager state.AlertInstanceManager, status StatusReader, store RuleStoreReader, authz RuleGroupAccessControlService) *PrometheusSrv { +type PrometheusSrv struct { + log log.Logger + manager state.AlertInstanceManager + status StatusReader + store RuleStoreReader + authz RuleGroupAccessControlService + provenanceStore ProvenanceStore +} + +func NewPrometheusSrv(log log.Logger, manager state.AlertInstanceManager, status StatusReader, store RuleStoreReader, authz RuleGroupAccessControlService, provenanceStore ProvenanceStore) *PrometheusSrv { return &PrometheusSrv{ - log: log, - manager: manager, - status: status, - store: store, - authz: authz, + log, + manager, + status, + store, + authz, + provenanceStore, } } @@ -274,12 +280,27 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon } } - ruleResponse = PrepareRuleGroupStatuses(srv.log, srv.store, RuleGroupStatusesOptions{ - Ctx: c.Req.Context(), - OrgID: c.OrgID, - Query: c.Req.Form, - AllowedNamespaces: allowedNamespaces, - }, RuleStatusMutatorGenerator(srv.status), RuleAlertStateMutatorGenerator(srv.manager)) + provenanceRecords, err := srv.provenanceStore.GetProvenances(c.Req.Context(), c.GetOrgID(), (&ngmodels.AlertRule{}).ResourceType()) + if err != nil { + ruleResponse.Status = "error" + ruleResponse.Error = fmt.Sprintf("failed to get provenances visible to the user: %s", err.Error()) + ruleResponse.ErrorType = apiv1.ErrServer + return response.JSON(ruleResponse.HTTPStatusCode(), ruleResponse) + } + + ruleResponse = PrepareRuleGroupStatuses( + srv.log, + srv.store, + RuleGroupStatusesOptions{ + Ctx: c.Req.Context(), + OrgID: c.OrgID, + Query: c.Req.Form, + AllowedNamespaces: allowedNamespaces, + }, + RuleStatusMutatorGenerator(srv.status), + RuleAlertStateMutatorGenerator(srv.manager), + provenanceRecords, + ) return response.JSON(ruleResponse.HTTPStatusCode(), ruleResponse) } @@ -379,7 +400,7 @@ func RuleAlertStateMutatorGenerator(manager state.AlertInstanceManager) RuleAler } } -func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts RuleGroupStatusesOptions, ruleStatusMutator RuleStatusMutator, alertStateMutator RuleAlertStateMutator) apimodels.RuleResponse { +func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts RuleGroupStatusesOptions, ruleStatusMutator RuleStatusMutator, alertStateMutator RuleAlertStateMutator, provenanceRecords map[string]ngmodels.Provenance) apimodels.RuleResponse { ruleResponse := apimodels.RuleResponse{ DiscoveryBase: apimodels.DiscoveryBase{ Status: "success", @@ -502,7 +523,7 @@ func PrepareRuleGroupStatuses(log log.Logger, store ListAlertRulesStore, opts Ru break } - ruleGroup, totals := toRuleGroup(log, rg.GroupKey, rg.Folder, rg.Rules, limitAlertsPerRule, stateFilterSet, matchers, labelOptions, ruleStatusMutator, alertStateMutator) + ruleGroup, totals := toRuleGroup(log, rg.GroupKey, rg.Folder, rg.Rules, provenanceRecords, limitAlertsPerRule, stateFilterSet, matchers, labelOptions, ruleStatusMutator, alertStateMutator) ruleGroup.Totals = totals for k, v := range totals { rulesTotals[k] += v @@ -653,7 +674,7 @@ func matchersMatch(matchers []*labels.Matcher, labels map[string]string) bool { return true } -func toRuleGroup(log log.Logger, groupKey ngmodels.AlertRuleGroupKey, folderFullPath string, rules []*ngmodels.AlertRule, limitAlerts int64, stateFilterSet map[eval.State]struct{}, matchers labels.Matchers, labelOptions []ngmodels.LabelOption, ruleStatusMutator RuleStatusMutator, ruleAlertStateMutator RuleAlertStateMutator) (*apimodels.RuleGroup, map[string]int64) { +func toRuleGroup(log log.Logger, groupKey ngmodels.AlertRuleGroupKey, folderFullPath string, rules []*ngmodels.AlertRule, provenanceRecords map[string]ngmodels.Provenance, limitAlerts int64, stateFilterSet map[eval.State]struct{}, matchers labels.Matchers, labelOptions []ngmodels.LabelOption, ruleStatusMutator RuleStatusMutator, ruleAlertStateMutator RuleAlertStateMutator) (*apimodels.RuleGroup, map[string]int64) { newGroup := &apimodels.RuleGroup{ Name: groupKey.RuleGroup, // file is what Prometheus uses for provisioning, we replace it with namespace which is the folder in Grafana. @@ -665,6 +686,10 @@ func toRuleGroup(log log.Logger, groupKey ngmodels.AlertRuleGroupKey, folderFull ngmodels.RulesGroup(rules).SortByGroupIndex() for _, rule := range rules { + provenance := ngmodels.ProvenanceNone + if prov, exists := provenanceRecords[rule.ResourceID()]; exists { + provenance = prov + } alertingRule := apimodels.AlertingRule{ State: "inactive", Name: rule.Title, @@ -674,12 +699,13 @@ func toRuleGroup(log log.Logger, groupKey ngmodels.AlertRuleGroupKey, folderFull KeepFiringFor: rule.KeepFiringFor.Seconds(), Annotations: apimodels.LabelsFromMap(rule.Annotations), Rule: apimodels.Rule{ - UID: rule.UID, - Name: rule.Title, - FolderUID: rule.NamespaceUID, - Labels: apimodels.LabelsFromMap(rule.GetLabels(labelOptions...)), - Type: rule.Type().String(), - IsPaused: rule.IsPaused, + UID: rule.UID, + Name: rule.Title, + FolderUID: rule.NamespaceUID, + Labels: apimodels.LabelsFromMap(rule.GetLabels(labelOptions...)), + Type: rule.Type().String(), + IsPaused: rule.IsPaused, + Provenance: apimodels.Provenance(provenance), }, } diff --git a/pkg/services/ngalert/api/tooling/definitions/prom.go b/pkg/services/ngalert/api/tooling/definitions/prom.go index 48a96f99c80..b11ef6fc62b 100644 --- a/pkg/services/ngalert/api/tooling/definitions/prom.go +++ b/pkg/services/ngalert/api/tooling/definitions/prom.go @@ -185,6 +185,7 @@ type Rule struct { EvaluationTime float64 `json:"evaluationTime"` IsPaused bool `json:"isPaused"` NotificationSettings *AlertRuleNotificationSettings `json:"notificationSettings,omitempty"` + Provenance Provenance `json:"provenance,omitempty"` } // Alert has info for an alert.