From 4bb6926eeea612cf266fa4b6de8c43481d981bd9 Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Wed, 9 Jul 2025 12:42:10 -0400 Subject: [PATCH] Alerting: Separate configuration model for remote Alertmanager Mimir client (#107741) * replace PostableUserConfig with GrafanaAlertmanagerConfig to decouple from internal Grafana models * update alertmanager + tests * calculate hash of the GrafanaAlertmanagerConfig --- pkg/services/ngalert/remote/alertmanager.go | 29 ++++++++----- .../ngalert/remote/alertmanager_test.go | 38 +++++++++++++++-- .../client/alertmanager_configuration.go | 28 +++++++++---- pkg/services/ngalert/remote/client/mimir.go | 2 +- pkg/services/ngalert/remote/compat.go | 13 ++++++ .../remote/test-data/config-with-extra.json | 41 +++++++++++++++++++ 6 files changed, 128 insertions(+), 23 deletions(-) create mode 100644 pkg/services/ngalert/remote/compat.go create mode 100644 pkg/services/ngalert/remote/test-data/config-with-extra.json diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 56cf9d8ec26..a5850cb4b0c 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -293,18 +293,20 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil { return fmt.Errorf("unable to merge extra configurations: %w", err) } - rawDecrypted, err := json.Marshal(decryptedCfg) + payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) + rawPayload, err := json.Marshal(payload) if err != nil { return fmt.Errorf("unable to marshal decrypted configuration: %w", err) } - configHash := md5.Sum(rawDecrypted) + + configHash := md5.Sum(rawPayload) // Send the configuration only if we need to. if !am.shouldSendConfig(ctx, configHash) { return nil } - return am.sendConfiguration(ctx, decryptedCfg, fmt.Sprintf("%x", configHash), config.CreatedAt, am.isDefaultConfiguration(configHash)) + return am.sendConfiguration(ctx, payload, fmt.Sprintf("%x", configHash), config.CreatedAt, am.isDefaultConfiguration(configHash)) } func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) bool { @@ -370,11 +372,11 @@ func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels return nil } -func (am *Alertmanager) sendConfiguration(ctx context.Context, decrypted *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error { +func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg *remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { am.metrics.ConfigSyncsTotal.Inc() if err := am.mimirClient.CreateGrafanaAlertmanagerConfig( ctx, - decrypted, + cfg, hash, createdAt, isDefault, @@ -420,14 +422,14 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil { return fmt.Errorf("unable to merge extra configurations: %w", err) } - - rawCfg, err := json.Marshal(decryptedCfg) + payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) + rawCfg, err := json.Marshal(payload) if err != nil { return err } hash := fmt.Sprintf("%x", md5.Sum(rawCfg)) - return am.sendConfiguration(ctx, decryptedCfg, hash, time.Now().Unix(), false) + return am.sendConfiguration(ctx, payload, hash, time.Now().Unix(), false) } // SaveAndApplyDefaultConfig sends the default Grafana Alertmanager configuration to the remote Alertmanager. @@ -446,10 +448,17 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error { return err } + payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) + rawCfg, err := json.Marshal(payload) + if err != nil { + return err + } + hash := fmt.Sprintf("%x", md5.Sum(rawCfg)) + return am.sendConfiguration( ctx, - decryptedCfg, - am.defaultConfigHash, + payload, + hash, time.Now().Unix(), true, ) diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index 3bbcb03de6b..27ec3f5a83d 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -3,6 +3,7 @@ package remote import ( "context" "crypto/md5" + "embed" "encoding/base64" "encoding/json" "errors" @@ -11,6 +12,7 @@ import ( "net/http" "net/http/httptest" "os" + "path" "slices" "strings" "testing" @@ -47,6 +49,9 @@ import ( "github.com/grafana/grafana/pkg/util" ) +//go:embed test-data/*.* +var testData embed.FS + var ( defaultGrafanaConfig = setting.GetAlertmanagerDefaultConfiguration() errTest = errors.New("test") @@ -357,12 +362,14 @@ func TestCompareAndSendConfiguration(t *testing.T) { testGrafanaConfigWithBadEncryption, err := json.Marshal(inputCfg) require.NoError(t, err) - cfgWithDecryptedSecret, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) + test, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) require.NoError(t, err) + cfgWithDecryptedSecret := PostableUserConfigToGrafanaAlertmanagerConfig(test) - cfgWithAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) + testAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) require.NoError(t, err) - require.NoError(t, testAutogenFn(nil, nil, 0, &cfgWithAutogenRoutes.AlertmanagerConfig, false)) + require.NoError(t, testAutogenFn(nil, nil, 0, &testAutogenRoutes.AlertmanagerConfig, false)) + cfgWithAutogenRoutes := PostableUserConfigToGrafanaAlertmanagerConfig(testAutogenRoutes) // Calculate hashes for expected configurations cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret) @@ -373,6 +380,20 @@ func TestCompareAndSendConfiguration(t *testing.T) { require.NoError(t, err) cfgWithAutogenRoutesHash := fmt.Sprintf("%x", md5.Sum(cfgWithAutogenRoutesBytes)) + cfgWithExtraUnmergedBytes, err := testData.ReadFile(path.Join("test-data", "config-with-extra.json")) + require.NoError(t, err) + cfgWithExtraUnmerged, err := notifier.Load(cfgWithExtraUnmergedBytes) + require.NoError(t, err) + r, err := cfgWithExtraUnmerged.GetMergedAlertmanagerConfig() + require.NoError(t, err) + cfgWithExtraMerged := &client.GrafanaAlertmanagerConfig{ + TemplateFiles: cfgWithExtraUnmerged.TemplateFiles, + AlertmanagerConfig: r.Config, + } + cfgWithExtraMergedBytes, err := json.Marshal(cfgWithExtraMerged) + require.NoError(t, err) + cfgWithExtraMergedHash := fmt.Sprintf("%x", md5.Sum(cfgWithExtraMergedBytes)) + tests := []struct { name string config string @@ -428,6 +449,15 @@ func TestCompareAndSendConfiguration(t *testing.T) { }, nil, }, + { + name: "no error, with extra configurations", + config: string(cfgWithExtraUnmergedBytes), + autogenFn: NoopAutogenFn, + expCfg: &client.UserGrafanaConfig{ + GrafanaAlertmanagerConfig: cfgWithExtraMerged, + Hash: cfgWithExtraMergedHash, + }, + }, } for _, test := range tests { @@ -677,7 +707,7 @@ func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) { // Return an empty config to ensure it gets replaced w.Header().Add("content-type", "application/json") require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{ - GrafanaAlertmanagerConfig: &apimodels.PostableUserConfig{}, + GrafanaAlertmanagerConfig: &client.GrafanaAlertmanagerConfig{}, })) return } diff --git a/pkg/services/ngalert/remote/client/alertmanager_configuration.go b/pkg/services/ngalert/remote/client/alertmanager_configuration.go index 164a3949011..35239700ea7 100644 --- a/pkg/services/ngalert/remote/client/alertmanager_configuration.go +++ b/pkg/services/ngalert/remote/client/alertmanager_configuration.go @@ -7,6 +7,7 @@ import ( "net/http" "github.com/grafana/alerting/definition" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ) @@ -15,14 +16,25 @@ const ( grafanaAlertmanagerReceiversPath = "/api/v1/grafana/receivers" ) +type GrafanaAlertmanagerConfig struct { + TemplateFiles map[string]string `yaml:"template_files" json:"template_files"` + AlertmanagerConfig definition.PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"` +} + +func (u *GrafanaAlertmanagerConfig) MarshalJSON() ([]byte, error) { + // this is special marshaling that makes sure that secrets are not masked + type cfg GrafanaAlertmanagerConfig + return definition.MarshalJSONWithSecrets((*cfg)(u)) +} + type UserGrafanaConfig struct { - GrafanaAlertmanagerConfig *apimodels.PostableUserConfig `json:"configuration"` - Hash string `json:"configuration_hash"` - CreatedAt int64 `json:"created"` - Default bool `json:"default"` - Promoted bool `json:"promoted"` - ExternalURL string `json:"external_url"` - SmtpConfig SmtpConfig `json:"smtp_config"` + GrafanaAlertmanagerConfig *GrafanaAlertmanagerConfig `json:"configuration"` + Hash string `json:"configuration_hash"` + CreatedAt int64 `json:"created"` + Default bool `json:"default"` + Promoted bool `json:"promoted"` + ExternalURL string `json:"external_url"` + SmtpConfig SmtpConfig `json:"smtp_config"` // TODO: Remove once everything can be sent in the 'SmtpConfig' field. SmtpFrom string `json:"smtp_from"` @@ -52,7 +64,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana return gc, nil } -func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error { +func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{ GrafanaAlertmanagerConfig: cfg, Hash: hash, diff --git a/pkg/services/ngalert/remote/client/mimir.go b/pkg/services/ngalert/remote/client/mimir.go index 931e2f7aa3d..61075dc1df4 100644 --- a/pkg/services/ngalert/remote/client/mimir.go +++ b/pkg/services/ngalert/remote/client/mimir.go @@ -29,7 +29,7 @@ type MimirClient interface { DeleteGrafanaAlertmanagerState(ctx context.Context) error GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) - CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error + CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error DeleteGrafanaAlertmanagerConfig(ctx context.Context) error TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error) diff --git a/pkg/services/ngalert/remote/compat.go b/pkg/services/ngalert/remote/compat.go new file mode 100644 index 00000000000..8914bc96728 --- /dev/null +++ b/pkg/services/ngalert/remote/compat.go @@ -0,0 +1,13 @@ +package remote + +import ( + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/remote/client" +) + +func PostableUserConfigToGrafanaAlertmanagerConfig(config *definitions.PostableUserConfig) *client.GrafanaAlertmanagerConfig { + return &client.GrafanaAlertmanagerConfig{ + TemplateFiles: config.TemplateFiles, + AlertmanagerConfig: config.AlertmanagerConfig, + } +} diff --git a/pkg/services/ngalert/remote/test-data/config-with-extra.json b/pkg/services/ngalert/remote/test-data/config-with-extra.json new file mode 100644 index 00000000000..4f15ddfc590 --- /dev/null +++ b/pkg/services/ngalert/remote/test-data/config-with-extra.json @@ -0,0 +1,41 @@ +{ + "template_files": { + "test": "{{ define \"my_templ\" }}TEST{{ end }}" + }, + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email", + "group_by": [ + "grafana_folder", + "alertname" + ] + }, + "receivers": [ + { + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "some other name", + "type": "email", + "disableResolveMessage": false, + "settings": { + "addresses": "" + } + } + ] + } + ] + }, + "extra_config": [ + { + "identifier": "imported", + "merge_matchers": ["imported=\"true\""], + "template_files": + { + "extra_template": "{{ define \"my_message\" }}TEST{{ end }}" + }, + "alertmanager_config": "{\"receivers\":[{\"webhook_configs\":[{\"url\":\"http://localhost\"}],\"name\":\"webhook\"}],\"route\":{\"receiver\":\"webhook\"}}" + } + ] +} \ No newline at end of file