Alerting: Support for simplified notification settings in rule API (#81011)

* Add notification settings to storage\domain and API models. Settings are a slice to workaround XORM mapping
* Support validation of notification settings when rules are updated

* Implement route generator for Alertmanager configuration. That fetches all notification settings.
* Update multi-tenant Alertmanager to run the generator before applying the configuration.

* Add notification settings labels to state calculation
* update the Multi-tenant Alertmanager to provide validation for notification settings

* update GET API so only admins can see auto-gen
This commit is contained in:
Yuri Tseretyan
2024-02-15 09:45:10 -05:00
committed by GitHub
parent ff916d9c15
commit 1eebd2a4de
60 changed files with 3466 additions and 304 deletions
+61 -15
View File
@@ -16,10 +16,12 @@ import (
"github.com/grafana/grafana/pkg/services/auth/identity"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/accesscontrol"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/quota"
@@ -33,6 +35,14 @@ type ConditionValidator interface {
Validate(ctx eval.EvaluationContext, condition ngmodels.Condition) error
}
type AMConfigStore interface {
GetLatestAlertmanagerConfiguration(ctx context.Context, orgID int64) (*ngmodels.AlertConfiguration, error)
}
type AMRefresher interface {
ApplyConfig(ctx context.Context, orgId int64, dbConfig *ngmodels.AlertConfiguration) error
}
type RulerSrv struct {
xactManager provisioning.TransactionManager
provenanceStore provisioning.ProvisioningStore
@@ -42,6 +52,10 @@ type RulerSrv struct {
cfg *setting.UnifiedAlertingSettings
conditionValidator ConditionValidator
authz RuleAccessControlService
amConfigStore AMConfigStore
amRefresher AMRefresher
featureManager featuremgmt.FeatureToggles
}
var (
@@ -283,8 +297,11 @@ func (srv RulerSrv) checkGroupLimits(group apimodels.PostableRuleGroupConfig) er
// updateAlertRulesInGroup calculates changes (rules to add,update,delete), verifies that the user is authorized to do the calculated changes and updates database.
// All operations are performed in a single transaction
//
//nolint:gocyclo
func (srv RulerSrv) updateAlertRulesInGroup(c *contextmodel.ReqContext, groupKey ngmodels.AlertRuleGroupKey, rules []*ngmodels.AlertRuleWithOptionals) response.Response {
var finalChanges *store.GroupDelta
var dbConfig *ngmodels.AlertConfiguration
err := srv.xactManager.InTransaction(c.Req.Context(), func(tranCtx context.Context) error {
userNamespace, id := c.SignedInUser.GetNamespacedID()
logger := srv.log.New("namespace_uid", groupKey.NamespaceUID, "group",
@@ -309,6 +326,24 @@ func (srv RulerSrv) updateAlertRulesInGroup(c *contextmodel.ReqContext, groupKey
return err
}
newOrUpdatedNotificationSettings := groupChanges.NewOrUpdatedNotificationSettings()
if len(newOrUpdatedNotificationSettings) > 0 {
dbConfig, err = srv.amConfigStore.GetLatestAlertmanagerConfiguration(c.Req.Context(), groupChanges.GroupKey.OrgID)
if err != nil {
return fmt.Errorf("failed to get latest configuration: %w", err)
}
cfg, err := notifier.Load([]byte(dbConfig.AlertmanagerConfiguration))
if err != nil {
return fmt.Errorf("failed to parse configuration: %w", err)
}
validator := notifier.NewNotificationSettingsValidator(&cfg.AlertmanagerConfig)
for _, s := range newOrUpdatedNotificationSettings {
if err := validator.Validate(s); err != nil {
return errors.Join(ngmodels.ErrAlertRuleFailedValidation, err)
}
}
}
if err := verifyProvisionedRulesNotAffected(c.Req.Context(), srv.provenanceStore, c.SignedInUser.GetOrgID(), groupChanges); err != nil {
return err
}
@@ -392,6 +427,15 @@ func (srv RulerSrv) updateAlertRulesInGroup(c *contextmodel.ReqContext, groupKey
}
return ErrResp(http.StatusInternalServerError, err, "failed to update rule group")
}
if srv.featureManager.IsEnabled(c.Req.Context(), featuremgmt.FlagAlertingSimplifiedRouting) && dbConfig != nil {
// This isn't strictly necessary since the alertmanager config is periodically synced.
err := srv.amRefresher.ApplyConfig(c.Req.Context(), groupKey.OrgID, dbConfig)
if err != nil {
srv.log.Warn("Failed to refresh Alertmanager config for org after change in notification settings", "org", c.SignedInUser.GetOrgID(), "error", err)
}
}
return changesToResponse(finalChanges)
}
@@ -440,23 +484,25 @@ func toGettableExtendedRuleNode(r ngmodels.AlertRule, provenanceRecords map[stri
if prov, exists := provenanceRecords[r.ResourceID()]; exists {
provenance = prov
}
gettableExtendedRuleNode := apimodels.GettableExtendedRuleNode{
GrafanaManagedAlert: &apimodels.GettableGrafanaRule{
ID: r.ID,
OrgID: r.OrgID,
Title: r.Title,
Condition: r.Condition,
Data: ApiAlertQueriesFromAlertQueries(r.Data),
Updated: r.Updated,
IntervalSeconds: r.IntervalSeconds,
Version: r.Version,
UID: r.UID,
NamespaceUID: r.NamespaceUID,
RuleGroup: r.RuleGroup,
NoDataState: apimodels.NoDataState(r.NoDataState),
ExecErrState: apimodels.ExecutionErrorState(r.ExecErrState),
Provenance: apimodels.Provenance(provenance),
IsPaused: r.IsPaused,
ID: r.ID,
OrgID: r.OrgID,
Title: r.Title,
Condition: r.Condition,
Data: ApiAlertQueriesFromAlertQueries(r.Data),
Updated: r.Updated,
IntervalSeconds: r.IntervalSeconds,
Version: r.Version,
UID: r.UID,
NamespaceUID: r.NamespaceUID,
RuleGroup: r.RuleGroup,
NoDataState: apimodels.NoDataState(r.NoDataState),
ExecErrState: apimodels.ExecutionErrorState(r.ExecErrState),
Provenance: apimodels.Provenance(provenance),
IsPaused: r.IsPaused,
NotificationSettings: AlertRuleNotificationSettingsFromNotificationSettings(r.NotificationSettings),
},
}
forDuration := model.Duration(r.For)