public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Fix loss of TimeInterval location on remote AM apply (#102510)

Browse Source 
* Alerting: Fix loss of TimeInterval location on remote AM apply

deepcopy.Copy does not correctly copy PostableUserConfig because it ignores
unexported fields. As a result, TimeInterval locations default to UTC instead
of retaining their original values.

* make update-workspace
This commit is contained in:
Matthew Jacobson
2025-03-20 09:54:33 +01:00
committed by GitHub
parent 7b8e5467bb
commit 371ea5cda7
4 changed files with 56 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -124,7 +124,7 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c //@grafana/identity-access-team
github.com/mocktools/go-smtp-mock/v2 v2.3.1 // @grafana/grafana-backend-group
github.com/modern-go/reflect2 v1.0.2 // @grafana/alerting-backend
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // @grafana/alerting-backend
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect; @grafana/alerting-backend
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // @grafana/grafana-operator-experience-squad
github.com/olekukonko/tablewriter v0.0.5 // @grafana/grafana-backend-group
github.com/openfga/api/proto v0.0.0-20250127102726-f9709139a369 // @grafana/identity-access-team
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
-21
View File
@@ -7,7 +7,6 @@ import (
"time"
"github.com/go-openapi/strfmt"
"github.com/mohae/deepcopy"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/common/model"
@@ -698,26 +697,6 @@ func (c *PostableUserConfig) validate() error {
return nil
}
// Decrypt returns a copy of the configuration struct with decrypted secure settings in receivers.
func (c *PostableUserConfig) Decrypt(decryptFn func(payload []byte) ([]byte, error)) (PostableUserConfig, error) {
newCfg, ok := deepcopy.Copy(c).(*PostableUserConfig)
if !ok {
return PostableUserConfig{}, fmt.Errorf("failed to copy config")
}
// Iterate through receivers and decrypt secure settings.
for _, rcv := range newCfg.AlertmanagerConfig.Receivers {
for _, gmr := range rcv.PostableGrafanaReceivers.GrafanaManagedReceivers {
decrypted, err := gmr.DecryptSecureSettings(decryptFn)
if err != nil {
return PostableUserConfig{}, err
}
gmr.SecureSettings = decrypted
}
}
return *newCfg, nil
}
// GetGrafanaReceiverMap returns a map that associates UUIDs to grafana receivers
func (c *PostableUserConfig) GetGrafanaReceiverMap() map[string]*PostableGrafanaReceiver {
UIDs := make(map[string]*PostableGrafanaReceiver)
pkg/services/ngalert/remote/alertmanager.go
+46 -26
View File
@@ -255,17 +255,22 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
return err
}
decrypted, err := am.decryptConfiguration(ctx, c)
rawDecrypted, configHash, err := am.decryptConfiguration(ctx, c)
if err != nil {
return err
}
// Send the configuration only if we need to.
if !am.shouldSendConfig(ctx, decrypted) {
if !am.shouldSendConfig(ctx, configHash) {
return nil
}
isDefault, err := am.isDefaultConfiguration(decrypted)
isDefault, err := am.isDefaultConfiguration(configHash)
if err != nil {
return err
}
decrypted, err := notifier.Load(rawDecrypted)
if err != nil {
return err
}
@@ -273,27 +278,37 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
return am.sendConfiguration(ctx, decrypted, config.ConfigurationHash, config.CreatedAt, isDefault)
}
func (am *Alertmanager) isDefaultConfiguration(cfg *apimodels.PostableUserConfig) (bool, error) {
rawCfg, err := json.Marshal(cfg)
if err != nil {
return false, err
}
configHash := fmt.Sprintf("%x", md5.Sum(rawCfg))
return configHash == am.defaultConfigHash, nil
func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) (bool, error) {
return fmt.Sprintf("%x", configHash) == am.defaultConfigHash, nil
}
func (am *Alertmanager) decryptConfiguration(ctx context.Context, cfg *apimodels.PostableUserConfig) (*apimodels.PostableUserConfig, error) {
// decryptConfiguration decrypts the configuration in-place and returns the decrypted configuration alongside its hash.
// Should not be used outside of this package and the specific use case of decrypting the configuration before sending
// it to the remote Alertmanager.
func (am *Alertmanager) decryptConfiguration(ctx context.Context, cfg *apimodels.PostableUserConfig) ([]byte, [16]byte, error) {
fn := func(payload []byte) ([]byte, error) {
return am.decrypt(ctx, payload)
}
decrypted, err := cfg.Decrypt(fn)
if err != nil {
return nil, fmt.Errorf("unable to decrypt the configuration: %w", err)
// Iterate through receivers and decrypt secure settings.
// It's not necessary to be careful about not modifying the original, as it's used only in a specific context where
// the config is read from json and then immediately sent to the remote Alertmanager.
for _, rcv := range cfg.AlertmanagerConfig.Receivers {
for _, gmr := range rcv.PostableGrafanaReceivers.GrafanaManagedReceivers {
decrypted, err := gmr.DecryptSecureSettings(fn)
if err != nil {
return nil, [16]byte{}, fmt.Errorf("unable to decrypt settings on receiver %q (uid: %q): %w", gmr.Name, gmr.UID, err)
}
gmr.SecureSettings = decrypted
}
}
return &decrypted, nil
rawDecrypted, err := json.Marshal(cfg)
if err != nil {
return nil, [16]byte{}, fmt.Errorf("unable to marshal decrypted configuration: %w", err)
}
return rawDecrypted, md5.Sum(rawDecrypted), nil
}
func (am *Alertmanager) sendConfiguration(ctx context.Context, decrypted *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
@@ -345,7 +360,12 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
if err := am.autogenFn(ctx, am.log, am.orgID, &cfg.AlertmanagerConfig, false); err != nil {
return err
}
decrypted, err := am.decryptConfiguration(ctx, cfg)
rawDecrypted, _, err := am.decryptConfiguration(ctx, cfg)
if err != nil {
return err
}
decrypted, err := notifier.Load(rawDecrypted)
if err != nil {
return err
}
@@ -364,7 +384,12 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
return err
}
decrypted, err := am.decryptConfiguration(ctx, c)
rawDecrypted, _, err := am.decryptConfiguration(ctx, c)
if err != nil {
return err
}
decrypted, err := notifier.Load(rawDecrypted)
if err != nil {
return err
}
@@ -634,7 +659,7 @@ func (am *Alertmanager) getFullState(ctx context.Context) (string, error) {
// shouldSendConfig compares the remote Alertmanager configuration with our local one.
// It returns true if the configurations are different.
func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *apimodels.PostableUserConfig) bool {
func (am *Alertmanager) shouldSendConfig(ctx context.Context, hash [16]byte) bool {
rc, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx)
if err != nil {
// Log the error and return true so we try to upload our config anyway.
@@ -651,12 +676,7 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *apimodels.
am.log.Error("Unable to marshal the remote Alertmanager configuration for comparison", "err", err)
return true
}
rawInternal, err := json.Marshal(config)
if err != nil {
am.log.Error("Unable to marshal the internal Alertmanager configuration for comparison", "err", err)
return true
}
return md5.Sum(rawRemote) != md5.Sum(rawInternal)
return md5.Sum(rawRemote) != hash
}
// shouldSendState compares the remote Alertmanager state with our local one.
pkg/services/ngalert/remote/alertmanager_test.go
+9 -6
View File
@@ -23,6 +23,8 @@ import (
"github.com/grafana/alerting/definition"
alertingModels "github.com/grafana/alerting/models"
"github.com/grafana/alerting/notify"
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -39,13 +41,12 @@ import (
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tests/testsuite"
"github.com/grafana/grafana/pkg/util"
"gopkg.in/yaml.v3"
)
const (
// Valid Grafana Alertmanager configurations.
testGrafanaConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"}}]}]}}`
testGrafanaConfigWithSecret = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"test"}}]}]}}`
testGrafanaConfig = `{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"}}]}]}}`
testGrafanaConfigWithSecret = `{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"test"}}]}]}}`
testGrafanaDefaultConfigWithDifferentFieldOrder = `{"alertmanager_config":{"route":{"group_by":["alertname","grafana_folder"],"receiver":"grafana-default-email"},"receivers":[{"grafana_managed_receiver_configs":[{"uid":"","name":"email receiver","type":"email","settings":{"addresses":"<example@email.com>"}}],"name":"grafana-default-email"}]}}`
// Valid Alertmanager state base64 encoded.
@@ -287,14 +288,14 @@ func TestCompareAndSendConfiguration(t *testing.T) {
strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, `"password":"!"`, 1),
NoopAutogenFn,
nil,
"unable to decrypt the configuration: failed to decode value for key 'password': illegal base64 data at input byte 0",
`unable to decrypt settings on receiver "WH" (uid: "dde6ntuob69dtf"): failed to decode value for key 'password': illegal base64 data at input byte 0`,
},
{
"decrypt error",
testGrafanaConfigWithSecret,
NoopAutogenFn,
nil,
fmt.Sprintf("unable to decrypt the configuration: failed to decrypt value for key 'password': %s", errTest.Error()),
fmt.Sprintf(`unable to decrypt settings on receiver "WH" (uid: "dde6ntuob69dtf"): failed to decrypt value for key 'password': %s`, errTest.Error()),
},
{
"error from autogen function",
@@ -443,7 +444,9 @@ func Test_isDefaultConfiguration(t *testing.T) {
defaultConfig: string(rawDefaultCfg),
defaultConfigHash: fmt.Sprintf("%x", md5.Sum(rawDefaultCfg)),
}
isDefault, _ := am.isDefaultConfiguration(test.config)
raw, err := json.Marshal(test.config)
require.NoError(tt, err)
isDefault, _ := am.isDefaultConfiguration(md5.Sum(raw))
require.Equal(tt, test.expected, isDefault)
})
}