Alerting: Remote Alertmanager to calculate hash of the request payload instead of just the configuration (#108632)

* update CreateGrafanaAlertmanagerConfig to accept UserGrafanaConfig move construction logic to alertmanager
* consolidate building UserGrafanaConfig into buildConfig
* use config to determine whether it needs to be send calculate hash of the entire request struct rather than configuration
This commit is contained in:
Yuri Tseretyan
2025-07-31 10:25:43 -04:00
committed by GitHub
parent 7d8c2fbb82
commit 32434810e1
4 changed files with 102 additions and 156 deletions
+65 -86
View File
@@ -2,10 +2,10 @@ package remote
import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"fmt"
"hash/fnv"
"net/http"
"net/url"
"strings"
@@ -16,12 +16,12 @@ import (
"github.com/grafana/alerting/definition"
alertingModels "github.com/grafana/alerting/models"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/alerting/utils/hash"
amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
amgeneral "github.com/prometheus/alertmanager/api/v2/client/general"
amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
"github.com/prometheus/client_golang/prometheus"
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/infra/log"
@@ -73,6 +73,9 @@ type Alertmanager struct {
amClient *remoteClient.Alertmanager
mimirClient remoteClient.MimirClient
promoteConfig bool
externalURL string
}
type AlertmanagerConfig struct {
@@ -127,13 +130,10 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
logger := log.New("ngalert.remote.alertmanager")
mcCfg := &remoteClient.Config{
Logger: logger,
Password: cfg.BasicAuthPassword,
TenantID: cfg.TenantID,
URL: u,
PromoteConfig: cfg.PromoteConfig,
ExternalURL: cfg.ExternalURL,
Smtp: cfg.SmtpConfig,
Logger: logger,
Password: cfg.BasicAuthPassword,
TenantID: cfg.TenantID,
URL: u,
}
mc, err := remoteClient.New(mcCfg, metrics, tracer)
if err != nil {
@@ -188,7 +188,10 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
syncInterval: cfg.SyncInterval,
tenantID: cfg.TenantID,
url: cfg.URL,
smtp: cfg.SmtpConfig,
externalURL: cfg.ExternalURL,
promoteConfig: cfg.PromoteConfig,
smtp: cfg.SmtpConfig,
}
// Parse the default configuration once and remember its hash so we can compare it later.
@@ -196,15 +199,11 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
// (grouping, group timing, time intervals etc) changes the autogenerated configuration.
// The `default` flag is sent to the remote Alertmanager for informational purposes, so we can tolerate this.
err = func() error {
defaultCfg, err := am.buildConfiguration(ctx, []byte(cfg.DefaultConfig))
defaultCfg, err := am.buildConfiguration(ctx, []byte(cfg.DefaultConfig), 0)
if err != nil {
return fmt.Errorf("unable to build default configuration: %w", err)
}
rawDefaultCfg, err := json.Marshal(defaultCfg)
if err != nil {
return fmt.Errorf("unable to marshal default configuration: %w", err)
}
am.defaultConfigHash = fmt.Sprintf("%x", md5.Sum(rawDefaultCfg))
am.defaultConfigHash = defaultCfg.Hash
return nil
}()
if err != nil {
@@ -265,22 +264,16 @@ func (am *Alertmanager) checkReadiness(ctx context.Context) error {
// CompareAndSendConfiguration checks whether a given configuration is being used by the remote Alertmanager.
// If not, it sends the configuration to the remote Alertmanager.
func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config *models.AlertConfiguration) error {
payload, err := am.buildConfiguration(ctx, []byte(config.AlertmanagerConfiguration))
payload, err := am.buildConfiguration(ctx, []byte(config.AlertmanagerConfiguration), config.CreatedAt)
if err != nil {
return fmt.Errorf("unable to build configuration: %w", err)
}
rawPayload, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
}
configHash := fmt.Sprintf("%x", md5.Sum(rawPayload))
// Send the configuration only if we need to.
if !am.shouldSendConfig(ctx, configHash) {
if !am.shouldSendConfig(ctx, payload.Hash) {
return nil
}
return am.sendConfiguration(ctx, payload, configHash, config.CreatedAt, am.isDefaultConfiguration(configHash))
return am.sendConfiguration(ctx, payload)
}
func (am *Alertmanager) isDefaultConfiguration(configHash string) bool {
@@ -303,31 +296,31 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
// buildConfiguration takes a raw Alertmanager configuration and returns a config that the remote Alertmanager can use.
// It parses the initial configuration, adds auto-generated routes, decrypts receivers, and merges the extra configs.
func (am *Alertmanager) buildConfiguration(ctx context.Context, raw []byte) (remoteClient.GrafanaAlertmanagerConfig, error) {
func (am *Alertmanager) buildConfiguration(ctx context.Context, raw []byte, createdAtEpoch int64) (remoteClient.UserGrafanaConfig, error) {
c, err := notifier.Load(raw)
if err != nil {
return remoteClient.GrafanaAlertmanagerConfig{}, err
return remoteClient.UserGrafanaConfig{}, err
}
// Add auto-generated routes and decrypt before comparing.
if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
return remoteClient.GrafanaAlertmanagerConfig{}, err
return remoteClient.UserGrafanaConfig{}, err
}
// Decrypt the receivers in the configuration.
decryptedReceivers, err := legacy_storage.DecryptedReceivers(c.AlertmanagerConfig.Receivers, decrypter(ctx, am.crypto))
if err != nil {
return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to decrypt receivers: %w", err)
return remoteClient.UserGrafanaConfig{}, fmt.Errorf("unable to decrypt receivers: %w", err)
}
c.AlertmanagerConfig.Receivers = decryptedReceivers
if err := am.crypto.DecryptExtraConfigs(ctx, c); err != nil {
return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to decrypt extra configs: %w", err)
return remoteClient.UserGrafanaConfig{}, fmt.Errorf("unable to decrypt extra configs: %w", err)
}
mergeResult, err := c.GetMergedAlertmanagerConfig()
if err != nil {
return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
return remoteClient.UserGrafanaConfig{}, fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
}
var templates []definition.PostableApiTemplate
@@ -335,22 +328,31 @@ func (am *Alertmanager) buildConfiguration(ctx context.Context, raw []byte) (rem
templates = definition.TemplatesMapToPostableAPITemplates(c.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)
}
return remoteClient.GrafanaAlertmanagerConfig{
TemplateFiles: c.TemplateFiles,
AlertmanagerConfig: mergeResult.Config,
Templates: templates,
}, nil
payload := remoteClient.UserGrafanaConfig{
GrafanaAlertmanagerConfig: remoteClient.GrafanaAlertmanagerConfig{
TemplateFiles: c.TemplateFiles,
AlertmanagerConfig: mergeResult.Config,
Templates: templates,
},
CreatedAt: createdAtEpoch,
Promoted: am.promoteConfig,
ExternalURL: am.externalURL,
SmtpConfig: am.smtp,
}
cfgHash, err := calculateUserGrafanaConfigHash(payload)
if err != nil {
am.log.Error("Unable to calculate hash of the configuration. Using the empty string", "error", err)
cfgHash = ""
}
payload.Hash = cfgHash
payload.Default = am.isDefaultConfiguration(cfgHash)
return payload, nil
}
func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg remoteClient.UserGrafanaConfig) error {
am.metrics.ConfigSyncsTotal.Inc()
if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(
ctx,
cfg,
hash,
createdAt,
isDefault,
); err != nil {
if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(ctx, &cfg); err != nil {
am.metrics.ConfigSyncErrorsTotal.Inc()
return err
}
@@ -422,40 +424,25 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
return err
}
payload, err := am.buildConfiguration(ctx, rawCopy)
payload, err := am.buildConfiguration(ctx, rawCopy, time.Now().Unix())
if err != nil {
return fmt.Errorf("unable to build configuration: %w", err)
}
rawCfg, err := json.Marshal(payload)
if err != nil {
return err
}
hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
return am.sendConfiguration(ctx, payload, hash, time.Now().Unix(), false)
return am.sendConfiguration(ctx, payload)
}
// SaveAndApplyDefaultConfig sends the default Grafana Alertmanager configuration to the remote Alertmanager.
func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
am.log.Debug("Sending default configuration to a remote Alertmanager", "url", am.url)
payload, err := am.buildConfiguration(ctx, []byte(am.defaultConfig))
payload, err := am.buildConfiguration(ctx, []byte(am.defaultConfig), time.Now().Unix())
if err != nil {
return fmt.Errorf("unable to build default configuration: %w", err)
}
rawCfg, err := json.Marshal(payload)
if err != nil {
return err
}
hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
payload.Default = true // override default status
return am.sendConfiguration(
ctx,
payload,
hash,
time.Now().Unix(),
true,
)
}
@@ -696,37 +683,29 @@ func (am *Alertmanager) getFullState(ctx context.Context) (string, error) {
// shouldSendConfig compares the remote Alertmanager configuration with our local one.
// It returns true if the configurations are different.
func (am *Alertmanager) shouldSendConfig(ctx context.Context, hash string) bool {
if hash == "" { // empty hash means that something went wrong while calculating it. In this case, always send the config.
return true
}
rc, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx)
if err != nil {
// Log the error and return true so we try to upload our config anyway.
am.log.Warn("Unable to get the remote Alertmanager configuration for comparison, sending the configuration without comparing", "err", err)
return true
}
if rc.Promoted != am.mimirClient.ShouldPromoteConfig() {
if rc.Hash != hash {
am.log.Debug("Hash of the remote Alertmanager configuration is different, sending the configuration", "remote", rc.Hash, "local", hash)
return true
}
return false
}
// Compare SMTP configs.
if rc.SmtpConfig.EhloIdentity != am.smtp.EhloIdentity ||
rc.SmtpConfig.Password != am.smtp.Password ||
rc.SmtpConfig.FromAddress != am.smtp.FromAddress ||
rc.SmtpConfig.FromName != am.smtp.FromName ||
rc.SmtpConfig.Host != am.smtp.Host ||
rc.SmtpConfig.SkipVerify != am.smtp.SkipVerify ||
rc.SmtpConfig.StartTLSPolicy != am.smtp.StartTLSPolicy ||
len(rc.SmtpConfig.StaticHeaders) != len(am.smtp.StaticHeaders) ||
rc.SmtpConfig.User != am.smtp.User {
am.log.Debug("SMTP config is different, sending the configuration to the remote Alertmanager")
return true
}
func calculateUserGrafanaConfigHash(config remoteClient.UserGrafanaConfig) (string, error) {
// Ignore some fields when calculating the hash. Make sure the original struct is not modified after that.
config.Default = false
config.CreatedAt = 0 // ignore createdAt to support comparison with hash of default config
config.Hash = ""
for k, v := range rc.SmtpConfig.StaticHeaders {
if value, ok := am.smtp.StaticHeaders[k]; !ok || v != value {
am.log.Debug("SMTP static headers are different, sending the configuration to the remote Alertmanager")
return true
}
}
return rc.Hash != hash
hasher := fnv.New64a()
hash.DeepHashObject(hasher, &config)
return fmt.Sprintf("%x", hasher.Sum64()), nil
}
@@ -19,10 +19,13 @@ import (
"time"
"github.com/go-openapi/strfmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/client_golang/prometheus"
common_config "github.com/prometheus/common/config"
"github.com/stretchr/testify/require"
alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
@@ -501,15 +504,6 @@ func TestCompareAndSendConfiguration(t *testing.T) {
AlertmanagerConfig: testAutogenRoutes.AlertmanagerConfig,
}
// Calculate hashes for expected configurations
cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret)
require.NoError(t, err)
cfgWithDecryptedSecretHash := fmt.Sprintf("%x", md5.Sum(cfgWithDecryptedSecretBytes))
cfgWithAutogenRoutesBytes, err := json.Marshal(cfgWithAutogenRoutes)
require.NoError(t, err)
cfgWithAutogenRoutesHash := fmt.Sprintf("%x", md5.Sum(cfgWithAutogenRoutesBytes))
cfgWithExtraUnmergedBytes, err := testData.ReadFile(path.Join("test-data", "config-with-extra.json"))
require.NoError(t, err)
cfgWithExtraUnmerged, err := notifier.Load(cfgWithExtraUnmergedBytes)
@@ -521,9 +515,6 @@ func TestCompareAndSendConfiguration(t *testing.T) {
AlertmanagerConfig: r.Config,
Templates: definition.TemplatesMapToPostableAPITemplates(cfgWithExtraUnmerged.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind),
}
cfgWithExtraMergedBytes, err := json.Marshal(cfgWithExtraMerged)
require.NoError(t, err)
cfgWithExtraMergedHash := fmt.Sprintf("%x", md5.Sum(cfgWithExtraMergedBytes))
tests := []struct {
name string
@@ -566,7 +557,6 @@ func TestCompareAndSendConfiguration(t *testing.T) {
NoopAutogenFn,
&client.UserGrafanaConfig{
GrafanaAlertmanagerConfig: cfgWithDecryptedSecret,
Hash: cfgWithDecryptedSecretHash,
},
nil,
},
@@ -576,7 +566,6 @@ func TestCompareAndSendConfiguration(t *testing.T) {
testAutogenFn,
&client.UserGrafanaConfig{
GrafanaAlertmanagerConfig: cfgWithAutogenRoutes,
Hash: cfgWithAutogenRoutesHash,
},
nil,
},
@@ -586,7 +575,6 @@ func TestCompareAndSendConfiguration(t *testing.T) {
autogenFn: NoopAutogenFn,
expCfg: &client.UserGrafanaConfig{
GrafanaAlertmanagerConfig: cfgWithExtraMerged,
Hash: cfgWithExtraMergedHash,
},
},
}
@@ -614,9 +602,26 @@ func TestCompareAndSendConfiguration(t *testing.T) {
err = am.CompareAndSendConfiguration(ctx, &cfg)
if len(test.expErrContains) == 0 {
require.NoError(tt, err)
rawCfg, err := json.Marshal(test.expCfg)
var gotCfg client.UserGrafanaConfig
require.NoError(tt, json.Unmarshal([]byte(got), &gotCfg))
require.NotEmpty(tt, gotCfg.Hash)
require.Empty(tt, cmp.Diff(test.expCfg, &gotCfg,
cmpopts.IgnoreFields(client.UserGrafanaConfig{}, "Hash"), // do not compare hashes because the config is processed slightly different: empty maps are nils.
cmpopts.EquateEmpty(),
cmpopts.IgnoreUnexported(
time.Location{},
labels.Matcher{},
common_config.ProxyConfig{})))
got1 := got
got = ""
err = am.CompareAndSendConfiguration(ctx, &cfg)
require.NoError(tt, err)
require.JSONEq(tt, string(rawCfg), got)
got2 := got
require.Equalf(tt, got1, got2, "Configuration is not idempotent")
return
}
for _, expErr := range test.expErrContains {
@@ -815,12 +820,7 @@ receivers:
require.NotNil(t, extraReceiver)
require.Len(t, extraReceiver.EmailConfigs, 1)
require.Equal(t, "alerts@grafana.com", extraReceiver.EmailConfigs[0].To)
// Verify the config hash
expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
require.NoError(t, err)
expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
require.Equal(t, expectedHash, configSent.Hash)
require.NotEmpty(t, configSent.Hash)
}
func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) {
@@ -934,10 +934,7 @@ receivers:
require.True(t, found)
// Verify the config hash
expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
require.NoError(t, err)
expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
require.Equal(t, expectedHash, configSent.Hash)
require.NotEmpty(t, configSent.Hash)
}
func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
@@ -961,11 +958,10 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
DefaultConfig: defaultGrafanaConfig,
}
testConfigHash := fmt.Sprintf("%x", md5.Sum([]byte(testGrafanaConfig)))
testConfigCreatedAt := time.Now().Unix()
testConfig := &ngmodels.AlertConfiguration{
AlertmanagerConfiguration: testGrafanaConfig,
ConfigurationHash: testConfigHash,
ConfigurationHash: "",
ConfigurationVersion: "v2",
CreatedAt: testConfigCreatedAt,
OrgID: 1,
@@ -1012,7 +1008,6 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
rawCfg, err := json.Marshal(config.GrafanaAlertmanagerConfig)
require.NoError(t, err)
require.JSONEq(t, testGrafanaConfig, string(rawCfg))
require.Equal(t, testConfigHash, config.Hash)
require.Equal(t, testConfigCreatedAt, config.CreatedAt)
require.Equal(t, testConfig.Default, config.Default)
@@ -1038,7 +1033,6 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
rawCfg, err := json.Marshal(config.GrafanaAlertmanagerConfig)
require.NoError(t, err)
require.JSONEq(t, testGrafanaConfig, string(rawCfg))
require.Equal(t, testConfigHash, config.Hash)
require.Equal(t, testConfigCreatedAt, config.CreatedAt)
require.False(t, config.Default)
@@ -1085,9 +1079,6 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
require.JSONEq(t, testGrafanaConfigWithSecret, string(got))
// Verify that the hash is calculated from the final configuration, including simplified routing
expectedHash := fmt.Sprintf("%x", md5.Sum(got))
require.Equal(t, expectedHash, config.Hash, "Hash should be calculated from the final processed configuration")
require.False(t, config.Default)
// An error while adding auto-generated rutes should be returned.
@@ -1114,7 +1105,6 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
require.NoError(t, err)
require.JSONEq(t, string(want), string(got))
require.Equal(t, fmt.Sprintf("%x", md5.Sum(want)), config.Hash)
require.True(t, config.Default)
// An error while adding auto-generated rutes should be returned.
@@ -39,10 +39,6 @@ type UserGrafanaConfig struct {
SmtpConfig SmtpConfig `json:"smtp_config"`
}
func (mc *Mimir) ShouldPromoteConfig() bool {
return mc.promoteConfig
}
func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) {
gc := &UserGrafanaConfig{}
response := successResponse{
@@ -62,16 +58,8 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana
return gc, nil
}
func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{
GrafanaAlertmanagerConfig: cfg,
Hash: hash,
CreatedAt: createdAt,
Default: isDefault,
Promoted: mc.promoteConfig,
ExternalURL: mc.externalURL,
SmtpConfig: mc.smtpConfig,
})
func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *UserGrafanaConfig) error {
payload, err := definition.MarshalJSONWithSecrets(cfg)
if err != nil {
return err
}
+10 -21
View File
@@ -30,26 +30,21 @@ type MimirClient interface {
DeleteGrafanaAlertmanagerState(ctx context.Context) error
GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error)
CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error
CreateGrafanaAlertmanagerConfig(ctx context.Context, config *UserGrafanaConfig) error
DeleteGrafanaAlertmanagerConfig(ctx context.Context) error
TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error)
TestReceivers(ctx context.Context, c alertingNotify.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error)
ShouldPromoteConfig() bool
// Mimir implements an extended version of the receivers API under a different path.
GetReceivers(ctx context.Context) ([]apimodels.Receiver, error)
}
type Mimir struct {
client client.Requester
endpoint *url.URL
logger log.Logger
metrics *metrics.RemoteAlertmanager
promoteConfig bool
externalURL string
smtpConfig SmtpConfig
client client.Requester
endpoint *url.URL
logger log.Logger
metrics *metrics.RemoteAlertmanager
}
type SmtpConfig struct {
@@ -69,10 +64,7 @@ type Config struct {
TenantID string
Password string
Logger log.Logger
PromoteConfig bool
ExternalURL string
Smtp SmtpConfig
Logger log.Logger
}
// successResponse represents a successful response from the Mimir API.
@@ -110,13 +102,10 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
return &Mimir{
endpoint: cfg.URL,
client: trc,
logger: cfg.Logger,
metrics: metrics,
promoteConfig: cfg.PromoteConfig,
externalURL: cfg.ExternalURL,
smtpConfig: cfg.Smtp,
endpoint: cfg.URL,
client: trc,
logger: cfg.Logger,
metrics: metrics,
}, nil
}