public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support for simplified notification settings in rule API (#81011)

Browse Source 
* Add notification settings to storage\domain and API models. Settings are a slice to workaround XORM mapping
* Support validation of notification settings when rules are updated

* Implement route generator for Alertmanager configuration. That fetches all notification settings.
* Update multi-tenant Alertmanager to run the generator before applying the configuration.

* Add notification settings labels to state calculation
* update the Multi-tenant Alertmanager to provide validation for notification settings

* update GET API so only admins can see auto-gen
This commit is contained in:
Yuri Tseretyan
2024-02-15 09:45:10 -05:00
committed by GitHub
parent ff916d9c15
commit 1eebd2a4de
60 changed files with 3466 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+46
View File
@@ -735,6 +735,18 @@ type GettableApiAlertingConfig struct {
Receivers []*GettableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
}
func (c *GettableApiAlertingConfig) GetReceivers() []*GettableApiReceiver {
return c.Receivers
}
func (c *GettableApiAlertingConfig) GetMuteTimeIntervals() []config.MuteTimeInterval {
return c.MuteTimeIntervals
}
func (c *GettableApiAlertingConfig) GetRoute() *Route {
return c.Route
}
func (c *GettableApiAlertingConfig) UnmarshalJSON(b []byte) error {
type plain GettableApiAlertingConfig
if err := json.Unmarshal(b, (*plain)(c)); err != nil {
@@ -960,6 +972,18 @@ type PostableApiAlertingConfig struct {
Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
}
func (c *PostableApiAlertingConfig) GetReceivers() []*PostableApiReceiver {
return c.Receivers
}
func (c *PostableApiAlertingConfig) GetMuteTimeIntervals() []config.MuteTimeInterval {
return c.MuteTimeIntervals
}
func (c *PostableApiAlertingConfig) GetRoute() *Route {
return c.Route
}
func (c *PostableApiAlertingConfig) UnmarshalJSON(b []byte) error {
type plain PostableApiAlertingConfig
if err := json.Unmarshal(b, (*plain)(c)); err != nil {
@@ -1047,6 +1071,12 @@ func AllReceivers(route *config.Route) (res []string) {
if route == nil {
return res
}
// TODO: Consider removing this check when new resource-specific AM APIs are implemented.
// Skip autogenerated routes. This helps cover the case where an admin POSTs the autogenerated route back to us.
// For example, when deleting a contact point that is unused but still referenced in the autogenerated route.
if isAutogeneratedRoot(route) {
return nil
}
if route.Receiver != "" {
res = append(res, route.Receiver)
@@ -1058,6 +1088,14 @@ func AllReceivers(route *config.Route) (res []string) {
return res
}
// autogeneratedRouteLabel a label name used to distinguish alerts that are supposed to be handled by the autogenerated policy. Only expected value is `true`.
const autogeneratedRouteLabel = "__grafana_autogenerated__"
// isAutogeneratedRoot returns true if the route is the root of an autogenerated route.
func isAutogeneratedRoot(route *config.Route) bool {
return len(route.Matchers) == 1 && route.Matchers[0].Name == autogeneratedRouteLabel
}
type RawMessage json.RawMessage // This type alias adds YAML marshaling to the json.RawMessage.
// MarshalJSON returns m as the JSON encoding of m.
@@ -1195,6 +1233,10 @@ func (r *GettableApiReceiver) Type() ReceiverType {
return AlertmanagerReceiverType
}
func (r *GettableApiReceiver) GetName() string {
return r.Receiver.Name
}
type PostableApiReceiver struct {
config.Receiver `yaml:",inline"`
PostableGrafanaReceivers `yaml:",inline"`
@@ -1263,6 +1305,10 @@ func (r *PostableApiReceiver) Type() ReceiverType {
return AlertmanagerReceiverType
}
func (r *PostableApiReceiver) GetName() string {
return r.Receiver.Name
}
type GettableGrafanaReceivers struct {
GrafanaManagedReceivers []*GettableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"`
}
pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go
+73
View File
@@ -8,6 +8,7 @@ import (
"testing"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -340,6 +341,78 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
},
err: true,
},
{
desc: "success undefined am receiver in autogenerated route is ignored",
input: PostableApiAlertingConfig{
Config: Config{
Route: &Route{
Receiver: "am",
Routes: []*Route{
{
Matchers: config.Matchers{
{
Name: autogeneratedRouteLabel,
Type: labels.MatchEqual,
Value: "true",
},
},
Routes: []*Route{
{
Receiver: "unmentioned",
},
},
},
},
},
},
Receivers: []*PostableApiReceiver{
{
Receiver: config.Receiver{
Name: "am",
EmailConfigs: []*config.EmailConfig{{}},
},
},
},
},
err: false,
},
{
desc: "success undefined graf receiver in autogenerated route is ignored",
input: PostableApiAlertingConfig{
Config: Config{
Route: &Route{
Receiver: "graf",
Routes: []*Route{
{
Matchers: config.Matchers{
{
Name: autogeneratedRouteLabel,
Type: labels.MatchEqual,
Value: "true",
},
},
Routes: []*Route{
{
Receiver: "unmentioned",
},
},
},
},
},
},
Receivers: []*PostableApiReceiver{
{
Receiver: config.Receiver{
Name: "graf",
},
PostableGrafanaReceivers: PostableGrafanaReceivers{
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{}},
},
},
},
},
err: false,
},
} {
t.Run(tc.desc, func(t *testing.T) {
encoded, err := json.Marshal(tc.input)
pkg/services/ngalert/api/tooling/definitions/alertmanager_validation.go
+4 -1
View File
@@ -13,12 +13,15 @@ import (
"gopkg.in/yaml.v3"
)
// groupByAll is a special value defined by alertmanager that can be used in a Route's GroupBy field to aggregate by all possible labels.
const groupByAll = "..."
// Validate normalizes a possibly nested Route r, and returns errors if r is invalid.
func (r *Route) validateChild() error {
r.GroupBy = nil
r.GroupByAll = false
for _, l := range r.GroupByStr {
if l == "..." {
if l == groupByAll {
r.GroupByAll = true
} else {
r.GroupBy = append(r.GroupBy, model.LabelName(l))
pkg/services/ngalert/api/tooling/definitions/cortex-ruler.go
+68 -22
View File
@@ -406,34 +406,80 @@ const (
ErrorErrState ExecutionErrorState = "Error"
)
// swagger: model
type AlertRuleNotificationSettings struct {
// Name of the receiver to send notifications to.
// required: true
// example: grafana-default-email
Receiver string `json:"receiver"`
// Optional settings
// Override the labels by which incoming alerts are grouped together. For example, multiple alerts coming in for
// cluster=A and alertname=LatencyHigh would be batched into a single group. To aggregate by all possible labels
// use the special value '...' as the sole label name.
// This effectively disables aggregation entirely, passing through all alerts as-is. This is unlikely to be what
// you want, unless you have a very low alert volume or your upstream notification system performs its own grouping.
// Must include 'alertname' and 'grafana_folder' if not using '...'.
// default: ["alertname", "grafana_folder"]
// example: ["alertname", "grafana_folder", "cluster"]
GroupBy []string `json:"group_by,omitempty"`
// Override how long to initially wait to send a notification for a group of alerts. Allows to wait for an
// inhibiting alert to arrive or collect more initial alerts for the same group. (Usually ~0s to few minutes.)
// example: 30s
GroupWait *model.Duration `json:"group_wait,omitempty"`
// Override how long to wait before sending a notification about new alerts that are added to a group of alerts for
// which an initial notification has already been sent. (Usually ~5m or more.)
// example: 5m
GroupInterval *model.Duration `json:"group_interval,omitempty"`
// Override how long to wait before sending a notification again if it has already been sent successfully for an
// alert. (Usually ~3h or more).
// Note that this parameter is implicitly bound by Alertmanager's `--data.retention` configuration flag.
// Notifications will be resent after either repeat_interval or the data retention period have passed, whichever
// occurs first. `repeat_interval` should not be less than `group_interval`.
// example: 4h
RepeatInterval *model.Duration `json:"repeat_interval,omitempty"`
// Override the times when notifications should be muted. These must match the name of a mute time interval defined
// in the alertmanager configuration mute_time_intervals section. When muted it will not send any notifications, but
// otherwise acts normally.
// example: ["maintenance"]
MuteTimeIntervals []string `json:"mute_time_intervals,omitempty"`
}
// swagger:model
type PostableGrafanaRule struct {
Title string `json:"title" yaml:"title"`
Condition string `json:"condition" yaml:"condition"`
Data []AlertQuery `json:"data" yaml:"data"`
UID string `json:"uid" yaml:"uid"`
NoDataState NoDataState `json:"no_data_state" yaml:"no_data_state"`
ExecErrState ExecutionErrorState `json:"exec_err_state" yaml:"exec_err_state"`
IsPaused *bool `json:"is_paused" yaml:"is_paused"`
Title string `json:"title" yaml:"title"`
Condition string `json:"condition" yaml:"condition"`
Data []AlertQuery `json:"data" yaml:"data"`
UID string `json:"uid" yaml:"uid"`
NoDataState NoDataState `json:"no_data_state" yaml:"no_data_state"`
ExecErrState ExecutionErrorState `json:"exec_err_state" yaml:"exec_err_state"`
IsPaused *bool `json:"is_paused" yaml:"is_paused"`
NotificationSettings *AlertRuleNotificationSettings `json:"notification_settings" yaml:"notification_settings"`
}
// swagger:model
type GettableGrafanaRule struct {
ID int64 `json:"id" yaml:"id"`
OrgID int64 `json:"orgId" yaml:"orgId"`
Title string `json:"title" yaml:"title"`
Condition string `json:"condition" yaml:"condition"`
Data []AlertQuery `json:"data" yaml:"data"`
Updated time.Time `json:"updated" yaml:"updated"`
IntervalSeconds int64 `json:"intervalSeconds" yaml:"intervalSeconds"`
Version int64 `json:"version" yaml:"version"`
UID string `json:"uid" yaml:"uid"`
NamespaceUID string `json:"namespace_uid" yaml:"namespace_uid"`
RuleGroup string `json:"rule_group" yaml:"rule_group"`
NoDataState NoDataState `json:"no_data_state" yaml:"no_data_state"`
ExecErrState ExecutionErrorState `json:"exec_err_state" yaml:"exec_err_state"`
Provenance Provenance `json:"provenance,omitempty" yaml:"provenance,omitempty"`
IsPaused bool `json:"is_paused" yaml:"is_paused"`
ID int64 `json:"id" yaml:"id"`
OrgID int64 `json:"orgId" yaml:"orgId"`
Title string `json:"title" yaml:"title"`
Condition string `json:"condition" yaml:"condition"`
Data []AlertQuery `json:"data" yaml:"data"`
Updated time.Time `json:"updated" yaml:"updated"`
IntervalSeconds int64 `json:"intervalSeconds" yaml:"intervalSeconds"`
Version int64 `json:"version" yaml:"version"`
UID string `json:"uid" yaml:"uid"`
NamespaceUID string `json:"namespace_uid" yaml:"namespace_uid"`
RuleGroup string `json:"rule_group" yaml:"rule_group"`
NoDataState NoDataState `json:"no_data_state" yaml:"no_data_state"`
ExecErrState ExecutionErrorState `json:"exec_err_state" yaml:"exec_err_state"`
Provenance Provenance `json:"provenance,omitempty" yaml:"provenance,omitempty"`
IsPaused bool `json:"is_paused" yaml:"is_paused"`
NotificationSettings *AlertRuleNotificationSettings `json:"notification_settings,omitempty" yaml:"notification_settings,omitempty"`
}
// AlertQuery represents a single query associated with an alert definition.
pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go
+18 -4
View File
@@ -156,6 +156,8 @@ type ProvisionedAlertRule struct {
Provenance Provenance `json:"provenance,omitempty"`
// example: false
IsPaused bool `json:"isPaused"`
// example: {"receiver":"email","group_by":["alertname","grafana_folder","cluster"],"group_wait":"30s","group_interval":"1m","repeat_interval":"4d","mute_time_intervals":["Weekends","Holidays"]}
NotificationSettings *AlertRuleNotificationSettings `json:"notification_settings"`
}
// swagger:route GET /v1/provisioning/folder/{FolderUID}/rule-groups/{Group} provisioning stable RouteGetAlertRuleGroup
@@ -246,10 +248,11 @@ type AlertRuleExport struct {
// ForString is used to:
// - Only export the for field for HCL if it is non-zero.
// - Format the Prometheus model.Duration type properly for HCL.
ForString *string `json:"-" yaml:"-" hcl:"for"`
Annotations *map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty" hcl:"annotations"`
Labels *map[string]string `json:"labels,omitempty" yaml:"labels,omitempty" hcl:"labels"`
IsPaused bool `json:"isPaused" yaml:"isPaused" hcl:"is_paused"`
ForString *string `json:"-" yaml:"-" hcl:"for"`
Annotations *map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty" hcl:"annotations"`
Labels *map[string]string `json:"labels,omitempty" yaml:"labels,omitempty" hcl:"labels"`
IsPaused bool `json:"isPaused" yaml:"isPaused" hcl:"is_paused"`
NotificationSettings *AlertRuleNotificationSettingsExport `json:"notification_settings,omitempty" yaml:"notification_settings,omitempty" hcl:"notification_settings,block"`
}
// AlertQueryExport is the provisioned export of models.AlertQuery.
@@ -266,3 +269,14 @@ type RelativeTimeRangeExport struct {
FromSeconds int64 `json:"from" yaml:"from" hcl:"from"`
ToSeconds int64 `json:"to" yaml:"to" hcl:"to"`
}
// AlertRuleNotificationSettingsExport is the provisioned export of models.NotificationSettings.
type AlertRuleNotificationSettingsExport struct {
Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty" hcl:"receiver"`
GroupBy []string `yaml:"group_by,omitempty" json:"group_by,omitempty" hcl:"group_by"`
GroupWait *string `yaml:"group_wait,omitempty" json:"group_wait,omitempty" hcl:"group_wait,optional"`
GroupInterval *string `yaml:"group_interval,omitempty" json:"group_interval,omitempty" hcl:"group_interval,optional"`
RepeatInterval *string `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty" hcl:"repeat_interval,optional"`
MuteTimeIntervals []string `yaml:"mute_time_intervals,omitempty" json:"mute_time_intervals,omitempty" hcl:"mute_time_intervals"`
}