public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Refactor Grafana Alertmanager (#105568)

Browse Source 
* update alerting module

* use NotificationsConfiguration
* update to use opts and configure new fields
* use TenantID instead of orgId
This commit is contained in:
Yuri Tseretyan
2025-05-20 04:40:51 +03:00
committed by GitHub
parent 2d71edbc52
commit 7dc13d63b9
5 changed files with 44 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -78,7 +78,7 @@ require (
github.com/googleapis/go-sql-spanner v1.11.1 // @grafana/grafana-search-and-storage
github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // @grafana/grafana-app-platform-squad
github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf // @grafana/alerting-backend
github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00 // @grafana/alerting-backend
github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb // @grafana/identity-access-team
github.com/grafana/authlib/types v0.0.0-20250325095148-d6da9c164a7d // @grafana/identity-access-team
github.com/grafana/dataplane/examples v0.0.1 // @grafana/observability-metrics
go.sum
+2 -2
View File
@@ -1569,8 +1569,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf h1:cbwO+P4La7EnDY1gdzFxsAf9u8ygM/7Laimuh7lYrNo=
github.com/grafana/alerting v0.0.0-20250513184957-2484dbaf55cf/go.mod h1:pMfhRxL2LZ3Pm8iy7VcVsb9CLYuBtjFYbf1oxgx7yFA=
github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00 h1:i0IpxkT1bX96F5D/oIxDdXLu+Nz/pU4rHlmnswpVLF0=
github.com/grafana/alerting v0.0.0-20250519195027-0ce55e7a1d00/go.mod h1:pMfhRxL2LZ3Pm8iy7VcVsb9CLYuBtjFYbf1oxgx7yFA=
github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb h1:oTl2j6/4miQUYmXANp2pBuYCWA5f8NVYFfCWpczpFso=
github.com/grafana/authlib v0.0.0-20250515162837-2f4a8263eabb/go.mod h1:PBtQaXwkFu4BAt2aXsR7w8p8NVpdjV5aJYhqRDei9Us=
github.com/grafana/authlib/types v0.0.0-20250325095148-d6da9c164a7d h1:34E6btDAhdDOiSEyrMaYaHwnJpM8w9QKzVQZIBzLNmM=
pkg/services/ngalert/notifier/alertmanager.go
+40 -71
View File
@@ -10,8 +10,6 @@ import (
"time"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/alerting/receivers"
alertingTemplates "github.com/grafana/alerting/templates"
"github.com/prometheus/alertmanager/config"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
@@ -51,14 +49,10 @@ type alertmanager struct {
Base *alertingNotify.GrafanaAlertmanager
logger log.Logger
ConfigMetrics *metrics.AlertmanagerConfigMetrics
Settings *setting.Cfg
Store AlertingStore
stateStore stateStore
NotificationService notifications.Service
decryptFn alertingNotify.GetDecryptedValueFn
orgID int64
ConfigMetrics *metrics.AlertmanagerConfigMetrics
Store AlertingStore
stateStore stateStore
DefaultConfiguration string
}
// maintenanceOptions represent the options for components that need maintenance on a frequency within the Alertmanager.
@@ -122,7 +116,7 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
}
l := log.New("ngalert.notifier.alertmanager", "org", orgID)
amcfg := &alertingNotify.GrafanaAlertmanagerConfig{
opts := alertingNotify.GrafanaAlertmanagerOpts{
ExternalURL: cfg.AppURL,
AlertStoreCallback: nil,
PeerTimeout: cfg.UnifiedAlerting.HAPeerTimeout,
@@ -132,23 +126,30 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
MaxSilences: cfg.UnifiedAlerting.AlertmanagerMaxSilencesCount,
MaxSilenceSizeBytes: cfg.UnifiedAlerting.AlertmanagerMaxSilenceSizeBytes,
},
EmailSender: &emailSender{ns},
ImageProvider: newImageProvider(store, log.New("ngalert.notifier.image-provider")),
Decrypter: decryptFn,
LoggerFactory: LoggerFactory,
Version: setting.BuildVersion,
TenantKey: "orgID",
TenantID: orgID,
Peer: peer,
Logger: l,
Metrics: alertingNotify.NewGrafanaAlertmanagerMetrics(m.Registerer, l),
}
gam, err := alertingNotify.NewGrafanaAlertmanager("orgID", orgID, amcfg, peer, l, alertingNotify.NewGrafanaAlertmanagerMetrics(m.Registerer, l))
gam, err := alertingNotify.NewGrafanaAlertmanager(opts)
if err != nil {
return nil, err
}
am := &alertmanager{
Base: gam,
ConfigMetrics: m.AlertmanagerConfigMetrics,
Settings: cfg,
Store: store,
NotificationService: ns,
orgID: orgID,
decryptFn: decryptFn,
stateStore: stateStore,
logger: l,
Base: gam,
ConfigMetrics: m.AlertmanagerConfigMetrics,
DefaultConfiguration: cfg.UnifiedAlerting.DefaultConfiguration,
Store: store,
stateStore: stateStore,
logger: l,
}
return am, nil
@@ -171,14 +172,14 @@ func (am *alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
var outerErr error
am.Base.WithLock(func() {
cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: am.Settings.UnifiedAlerting.DefaultConfiguration,
AlertmanagerConfiguration: am.DefaultConfiguration,
Default: true,
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: am.orgID,
OrgID: am.Base.TenantID(),
LastApplied: time.Now().UTC().Unix(),
}
cfg, err := Load([]byte(am.Settings.UnifiedAlerting.DefaultConfiguration))
cfg, err := Load([]byte(am.DefaultConfiguration))
if err != nil {
outerErr = err
return
@@ -213,7 +214,7 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: string(rawConfig),
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: am.orgID,
OrgID: am.Base.TenantID(),
LastApplied: time.Now().UTC().Unix(),
}
@@ -254,7 +255,7 @@ func (am *alertmanager) ApplyConfig(ctx context.Context, dbCfg *ngmodels.AlertCo
return
}
markConfigCmd := ngmodels.MarkConfigurationAsAppliedCmd{
OrgID: am.orgID,
OrgID: am.Base.TenantID(),
ConfigurationHash: dbCfg.ConfigurationHash,
}
err = am.Store.MarkConfigurationAsApplied(ctx, &markConfigCmd)
@@ -283,11 +284,11 @@ func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig, c
am.ConfigMetrics.ObjectMatchers.Set(float64(amu.ObjectMatchers))
am.ConfigMetrics.ConfigHash.
WithLabelValues(strconv.FormatInt(am.orgID, 10)).
WithLabelValues(strconv.FormatInt(am.Base.TenantID(), 10)).
Set(hashAsMetricValue(am.Base.ConfigHash()))
am.ConfigMetrics.ConfigSizeBytes.
WithLabelValues(strconv.FormatInt(am.orgID, 10)).
WithLabelValues(strconv.FormatInt(am.Base.TenantID(), 10)).
Set(float64(cfgSize))
}
@@ -316,7 +317,7 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu
// It returns a boolean indicating whether the user config was changed and an error.
// It is not safe to call concurrently.
func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, skipInvalid bool) (bool, error) {
err := AddAutogenConfig(ctx, am.logger, am.Store, am.orgID, &cfg.AlertmanagerConfig, skipInvalid)
err := AddAutogenConfig(ctx, am.logger, am.Store, am.Base.TenantID(), &cfg.AlertmanagerConfig, skipInvalid)
if err != nil {
return false, err
}
@@ -336,16 +337,16 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable
}
am.logger.Info("Applying new configuration to Alertmanager", "configHash", fmt.Sprintf("%x", configHash))
err = am.Base.ApplyConfig(AlertingConfiguration{
rawAlertmanagerConfig: rawConfig,
configHash: configHash,
route: cfg.AlertmanagerConfig.Route.AsAMRoute(),
inhibitRules: cfg.AlertmanagerConfig.InhibitRules,
muteTimeIntervals: cfg.AlertmanagerConfig.MuteTimeIntervals,
timeIntervals: cfg.AlertmanagerConfig.TimeIntervals,
templates: ToTemplateDefinitions(cfg),
receivers: PostableApiAlertingConfigToApiReceivers(cfg.AlertmanagerConfig),
receiverIntegrationsFunc: am.buildReceiverIntegrations,
err = am.Base.ApplyConfig(alertingNotify.NotificationsConfiguration{
RoutingTree: cfg.AlertmanagerConfig.Route.AsAMRoute(),
InhibitRules: cfg.AlertmanagerConfig.InhibitRules,
MuteTimeIntervals: cfg.AlertmanagerConfig.MuteTimeIntervals,
TimeIntervals: cfg.AlertmanagerConfig.TimeIntervals,
Templates: ToTemplateDefinitions(cfg),
Receivers: PostableApiAlertingConfigToApiReceivers(cfg.AlertmanagerConfig),
DispatcherLimits: &nilLimits{},
Raw: rawConfig,
Hash: configHash,
})
if err != nil {
return false, err
@@ -355,38 +356,6 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable
return true, nil
}
func (am *alertmanager) AppURL() string {
return am.Settings.AppURL
}
// buildReceiverIntegrations builds a list of integration notifiers off of a receiver config.
func (am *alertmanager) buildReceiverIntegrations(receiver *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) {
receiverCfg, err := alertingNotify.BuildReceiverConfiguration(context.Background(), receiver, alertingNotify.DecodeSecretsFromBase64, am.decryptFn)
if err != nil {
return nil, err
}
s := &emailSender{am.NotificationService}
img := newImageProvider(am.Store, log.New("ngalert.notifier.image-provider"))
integrations, err := alertingNotify.BuildReceiverIntegrations(
receiverCfg,
tmpl,
img,
LoggerFactory,
func(n receivers.Metadata) (receivers.EmailSender, error) {
return s, nil
},
func(_ string, n alertingNotify.Notifier) alertingNotify.Notifier {
return n
},
am.orgID,
setting.BuildVersion,
)
if err != nil {
return nil, err
}
return integrations, nil
}
// PutAlerts receives the alerts and then sends them through the corresponding route based on whenever the alert has a receiver embedded or not
func (am *alertmanager) PutAlerts(_ context.Context, postableAlerts apimodels.PostableAlerts) error {
alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts))
pkg/services/ngalert/notifier/config.go
-60
View File
@@ -4,9 +4,6 @@ import (
"encoding/json"
"fmt"
alertingNotify "github.com/grafana/alerting/notify"
alertingTemplates "github.com/grafana/alerting/templates"
api "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
)
@@ -19,60 +16,3 @@ func Load(rawConfig []byte) (*api.PostableUserConfig, error) {
return cfg, nil
}
// AlertingConfiguration provides configuration for an Alertmanager.
// It implements the notify.Configuration interface.
type AlertingConfiguration struct {
route *alertingNotify.Route
inhibitRules []alertingNotify.InhibitRule
muteTimeIntervals []alertingNotify.MuteTimeInterval
timeIntervals []alertingNotify.TimeInterval
templates []alertingTemplates.TemplateDefinition
rawAlertmanagerConfig []byte
configHash [16]byte
receivers []*alertingNotify.APIReceiver
receiverIntegrationsFunc func(r *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error)
}
func (a AlertingConfiguration) BuildReceiverIntegrationsFunc() func(next *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) {
return func(next *alertingNotify.APIReceiver, tmpl *alertingTemplates.Template) ([]*alertingNotify.Integration, error) {
return a.receiverIntegrationsFunc(next, tmpl)
}
}
func (a AlertingConfiguration) DispatcherLimits() alertingNotify.DispatcherLimits {
return &nilLimits{}
}
func (a AlertingConfiguration) InhibitRules() []alertingNotify.InhibitRule {
return a.inhibitRules
}
func (a AlertingConfiguration) MuteTimeIntervals() []alertingNotify.MuteTimeInterval {
return a.muteTimeIntervals
}
func (a AlertingConfiguration) TimeIntervals() []alertingNotify.TimeInterval {
return a.timeIntervals
}
func (a AlertingConfiguration) Receivers() []*alertingNotify.APIReceiver {
return a.receivers
}
func (a AlertingConfiguration) RoutingTree() *alertingNotify.Route {
return a.route
}
func (a AlertingConfiguration) Templates() []alertingTemplates.TemplateDefinition {
return a.templates
}
func (a AlertingConfiguration) Hash() [16]byte {
return a.configHash
}
func (a AlertingConfiguration) Raw() []byte {
return a.rawAlertmanagerConfig
}
pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
+1 -1
View File
@@ -221,7 +221,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
status, err := am.GetStatus(ctx)
require.NoError(t, err)
require.Equal(t, "N/A", *status.VersionInfo.Version)
require.Equal(t, int64(2), internalAm.orgID)
require.Equal(t, int64(2), internalAm.Base.TenantID())
}
// Let's now remove the previous queried organization.