From c9bb18101c8e97dca23fba35bdbb8fe36877bb70 Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 19 Mar 2024 12:12:03 +0100 Subject: [PATCH] Alerting: Decrypt secrets before sending configuration to the remote Alertmanager (#83640) * (WIP) Alerting: Decrypt secrets before sending configuration to the remote Alertmanager * refactor, fix tests * test decrypting secrets * tidy up * test SendConfiguration, quote keys, refactor tests * make linter happy * decrypt configuration before comparing * copy configuration struct before decrypting * reduce diff in TestCompareAndSendConfiguration * clean up remote/alertmanager.go * make linter happy * avoid serializing into JSON to copy struct * codeowners --- go.mod | 2 +- go.work.sum | 2 + .../api/tooling/definitions/alertmanager.go | 30 ++++ pkg/services/ngalert/ngalert.go | 6 +- .../multiorg_alertmanager_remote_test.go | 4 +- pkg/services/ngalert/remote/alertmanager.go | 61 +++++-- .../ngalert/remote/alertmanager_test.go | 162 ++++++++++++++++-- 7 files changed, 233 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 31149a5745e..4e8cd53c1f4 100644 --- a/go.mod +++ b/go.mod @@ -341,7 +341,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 //@grafana/grafana-authnz-team github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // @grafana/alerting-squad-backend github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20220512140940-7b36cea86235 // indirect diff --git a/go.work.sum b/go.work.sum index 6c3d23663fd..5c7308963ec 100644 --- a/go.work.sum +++ b/go.work.sum @@ -401,6 +401,8 @@ github.com/grafana/e2e v0.1.1-0.20221018202458-cffd2bb71c7b h1:Ha+kSIoTutf4ytlVw github.com/grafana/e2e v0.1.1-0.20221018202458-cffd2bb71c7b/go.mod h1:3UsooRp7yW5/NJQBlXcTsAHOoykEhNUYXkQ3r6ehEEY= github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 h1:/of8Z8taCPftShATouOrBVy6GaTTjgQd/VfNiZp/VXQ= github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= +github.com/grafana/grafana-aws-sdk v0.25.0 h1:XNi3iA/C/KPArmVbQfbwKQROaIotd38nCRjNE6P1UP0= +github.com/grafana/grafana-aws-sdk v0.25.0/go.mod h1:3zghFF6edrxn0d6k6X9HpGZXDH+VfA+MwD2Pc/9X0ec= github.com/grafana/grafana-plugin-sdk-go v0.212.0/go.mod h1:qsI4ktDf0lig74u8SLPJf9zRdVxWV/W4Wi+Ox6gifgs= github.com/grafana/grafana/pkg/promlib v0.0.2/go.mod h1:3El4NlsfALz8QQCbEGHGFvJUG+538QLMuALRhZ3pcoo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go index 3a9b6cc09b4..5248a2e5bbf 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go @@ -2,12 +2,14 @@ package definitions import ( "context" + "encoding/base64" "encoding/json" "fmt" "time" "github.com/go-openapi/strfmt" "github.com/grafana/alerting/definition" + "github.com/mohae/deepcopy" amv2 "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/config" "github.com/prometheus/common/model" @@ -615,6 +617,34 @@ func (c *PostableUserConfig) validate() error { return nil } +// Decrypt returns a copy of the configuration struct with decrypted secure settings in receivers. +func (c *PostableUserConfig) Decrypt(decryptFn func(payload []byte) ([]byte, error)) (PostableUserConfig, error) { + newCfg, ok := deepcopy.Copy(c).(*PostableUserConfig) + if !ok { + return PostableUserConfig{}, fmt.Errorf("failed to copy config") + } + + // Iterate through receivers and decrypt secure settings. + for _, rcv := range newCfg.AlertmanagerConfig.Receivers { + for _, gmr := range rcv.PostableGrafanaReceivers.GrafanaManagedReceivers { + for k, v := range gmr.SecureSettings { + decoded, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return PostableUserConfig{}, fmt.Errorf("failed to decode value for key '%s': %w", k, err) + } + + decrypted, err := decryptFn(decoded) + if err != nil { + return PostableUserConfig{}, fmt.Errorf("failed to decrypt value for key '%s': %w", k, err) + } + + gmr.SecureSettings[k] = string(decrypted) + } + } + } + return *newCfg, nil +} + // GetGrafanaReceiverMap returns a map that associates UUIDs to grafana receivers func (c *PostableUserConfig) GetGrafanaReceiverMap() map[string]*PostableGrafanaReceiver { UIDs := make(map[string]*PostableGrafanaReceiver) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 39714f9f6c6..0547084a667 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -190,7 +190,7 @@ func (ng *AlertNG) init() error { } // Create remote Alertmanager. - remoteAM, err := createRemoteAlertmanager(orgID, ng.Cfg.UnifiedAlerting.RemoteAlertmanager, ng.KVStore, m) + remoteAM, err := createRemoteAlertmanager(orgID, ng.Cfg.UnifiedAlerting.RemoteAlertmanager, ng.KVStore, ng.SecretsService.Decrypt, m) if err != nil { moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err) return internalAM, nil @@ -535,7 +535,7 @@ func ApplyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySet } } -func createRemoteAlertmanager(orgID int64, amCfg setting.RemoteAlertmanagerSettings, kvstore kvstore.KVStore, m *metrics.RemoteAlertmanager) (*remote.Alertmanager, error) { +func createRemoteAlertmanager(orgID int64, amCfg setting.RemoteAlertmanagerSettings, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, m *metrics.RemoteAlertmanager) (*remote.Alertmanager, error) { externalAMCfg := remote.AlertmanagerConfig{ OrgID: orgID, URL: amCfg.URL, @@ -544,5 +544,5 @@ func createRemoteAlertmanager(orgID int64, amCfg setting.RemoteAlertmanagerSetti } // We won't be handling files on disk, we can pass an empty string as workingDirPath. stateStore := notifier.NewFileStore(orgID, kvstore, "") - return remote.NewAlertmanager(externalAMCfg, stateStore, m) + return remote.NewAlertmanager(externalAMCfg, stateStore, decryptFn, m) } diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go index 603cfb7e654..a4a8c98a783 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go @@ -49,6 +49,7 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) { }) // Create the factory function for the MOA using the forked Alertmanager in remote secondary mode. + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) override := notifier.WithAlertmanagerOverride(func(factoryFn notifier.OrgAlertmanagerFactory) notifier.OrgAlertmanagerFactory { return func(ctx context.Context, orgID int64) (notifier.Alertmanager, error) { // Create internal Alertmanager. @@ -65,7 +66,7 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) { // We won't be handling files on disk, we can pass an empty string as workingDirPath. stateStore := notifier.NewFileStore(orgID, kvStore, "") m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - remoteAM, err := remote.NewAlertmanager(externalAMCfg, stateStore, m) + remoteAM, err := remote.NewAlertmanager(externalAMCfg, stateStore, secretsService.Decrypt, m) require.NoError(t, err) // Use both Alertmanager implementations in the forked Alertmanager. @@ -87,7 +88,6 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) { DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(), }, // do not poll in tests. } - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) moa, err := notifier.NewMultiOrgAlertmanager( cfg, configStore, diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 8eabc86fbe4..496bd26c532 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -3,6 +3,7 @@ package remote import ( "context" "crypto/md5" + "encoding/json" "fmt" "net/http" "net/url" @@ -25,7 +26,11 @@ type stateStore interface { GetFullState(ctx context.Context, keys ...string) (string, error) } +// DecryptFn is a function that takes in an encrypted value and returns it decrypted. +type DecryptFn func(ctx context.Context, payload []byte) ([]byte, error) + type Alertmanager struct { + decrypt DecryptFn log log.Logger metrics *metrics.RemoteAlertmanager orgID int64 @@ -61,7 +66,7 @@ func (cfg *AlertmanagerConfig) Validate() error { return nil } -func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) { +func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) { if err := cfg.Validate(); err != nil { return nil, err } @@ -111,6 +116,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, metrics *metrics. return &Alertmanager{ amClient: amc, + decrypt: decryptFn, log: logger, metrics: metrics, mimirClient: mc, @@ -176,21 +182,42 @@ func (am *Alertmanager) checkReadiness(ctx context.Context) error { // CompareAndSendConfiguration checks whether a given configuration is being used by the remote Alertmanager. // If not, it sends the configuration to the remote Alertmanager. func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config *models.AlertConfiguration) error { - if am.shouldSendConfig(ctx, config) { - am.metrics.ConfigSyncsTotal.Inc() - if err := am.mimirClient.CreateGrafanaAlertmanagerConfig( - ctx, - config.AlertmanagerConfiguration, - config.ConfigurationHash, - config.ID, - config.CreatedAt, - config.Default, - ); err != nil { - am.metrics.ConfigSyncErrorsTotal.Inc() - return err - } - am.metrics.LastConfigSync.SetToCurrentTime() + c, err := notifier.Load([]byte(config.AlertmanagerConfiguration)) + if err != nil { + return err } + + // Decrypt the configuration before comparing. + fn := func(payload []byte) ([]byte, error) { + return am.decrypt(ctx, payload) + } + decrypted, err := c.Decrypt(fn) + if err != nil { + return err + } + rawDecrypted, err := json.Marshal(decrypted) + if err != nil { + return err + } + + // Send the configuration only if we need to. + if !am.shouldSendConfig(ctx, rawDecrypted) { + return nil + } + + am.metrics.ConfigSyncsTotal.Inc() + if err := am.mimirClient.CreateGrafanaAlertmanagerConfig( + ctx, + string(rawDecrypted), + config.ConfigurationHash, + config.ID, + config.CreatedAt, + config.Default, + ); err != nil { + am.metrics.ConfigSyncErrorsTotal.Inc() + return err + } + am.metrics.LastConfigSync.SetToCurrentTime() return nil } @@ -375,7 +402,7 @@ func (am *Alertmanager) CleanUp() {} // shouldSendConfig compares the remote Alertmanager configuration with our local one. // It returns true if the configurations are different. -func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *models.AlertConfiguration) bool { +func (am *Alertmanager) shouldSendConfig(ctx context.Context, rawConfig []byte) bool { rc, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx) if err != nil { // Log the error and return true so we try to upload our config anyway. @@ -383,7 +410,7 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *models.Ale return true } - return md5.Sum([]byte(rc.GrafanaAlertmanagerConfig)) != md5.Sum([]byte(config.AlertmanagerConfiguration)) + return md5.Sum([]byte(rc.GrafanaAlertmanagerConfig)) != md5.Sum(rawConfig) } // shouldSendState compares the remote Alertmanager state with our local one. diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index 59e60034677..a500fcbf279 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -4,20 +4,31 @@ import ( "context" "crypto/md5" "encoding/base64" + "encoding/json" + "errors" "fmt" + "io" "math/rand" "net/http" "net/http/httptest" "os" + "strings" "testing" "time" "github.com/go-openapi/strfmt" + "github.com/grafana/grafana/pkg/infra/db" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/metrics" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/notifier" - "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/grafana/grafana/pkg/services/ngalert/remote/client" + ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/database" + "github.com/grafana/grafana/pkg/services/secrets/fakes" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/grafana/grafana/pkg/tests/testsuite" "github.com/grafana/grafana/pkg/util" amv2 "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/cluster/clusterpb" @@ -25,8 +36,13 @@ import ( "github.com/stretchr/testify/require" ) -// Valid Grafana Alertmanager configuration. +// Valid Grafana Alertmanager configurations. const testGrafanaConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}` +const testGrafanaConfigWithSecret = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"test"}}]}]}}` + +func TestMain(m *testing.M) { + testsuite.Run(m) +} func TestNewAlertmanager(t *testing.T) { tests := []struct { @@ -62,6 +78,7 @@ func TestNewAlertmanager(t *testing.T) { }, } + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) for _, test := range tests { t.Run(test.name, func(tt *testing.T) { cfg := AlertmanagerConfig{ @@ -71,7 +88,7 @@ func TestNewAlertmanager(t *testing.T) { BasicAuthPassword: test.password, } m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, nil, m) + am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, m) if test.expErr != "" { require.EqualError(tt, err, test.expErr) return @@ -90,10 +107,32 @@ func TestApplyConfig(t *testing.T) { errorHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }) + + var configSent string okHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/config") { + var c client.UserGrafanaConfig + require.NoError(t, json.NewDecoder(r.Body).Decode(&c)) + configSent = c.GrafanaAlertmanagerConfig + } + w.WriteHeader(http.StatusOK) }) + // Encrypt receivers to save secrets in the database. + var c apimodels.PostableUserConfig + require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &c)) + secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t))) + err := notifier.EncryptReceiverConfigs(c.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) { + return secretsService.Encrypt(ctx, payload, secrets.WithoutScope()) + }) + require.NoError(t, err) + + // The encrypted configuration should be different than the one we will send. + encryptedConfig, err := json.Marshal(c) + require.NoError(t, err) + require.NotEqual(t, testGrafanaConfig, encryptedConfig) + // ApplyConfig performs a readiness check at startup. // A non-200 response should result in an error. server := httptest.NewServer(errorHandler) @@ -104,16 +143,19 @@ func TestApplyConfig(t *testing.T) { } ctx := context.Background() - store := fakes.NewFakeKVStore(t) + store := ngfakes.NewFakeKVStore(t) fstore := notifier.NewFileStore(1, store, "") require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, "test")) require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, "test")) + // An error response from the remote Alertmanager should result in the readiness check failing. m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, fstore, m) + am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, m) require.NoError(t, err) - config := &ngmodels.AlertConfiguration{} + config := &ngmodels.AlertConfiguration{ + AlertmanagerConfiguration: string(encryptedConfig), + } require.Error(t, am.ApplyConfig(ctx, config)) require.False(t, am.Ready()) @@ -122,12 +164,104 @@ func TestApplyConfig(t *testing.T) { require.NoError(t, am.ApplyConfig(ctx, config)) require.True(t, am.Ready()) + // Secrets in the sent configuration should be unencrypted. + require.JSONEq(t, testGrafanaConfigWithSecret, configSent) + // If we already got a 200 status code response, we shouldn't make the HTTP request again. server.Config.Handler = errorHandler require.NoError(t, am.ApplyConfig(ctx, config)) require.True(t, am.Ready()) } +func TestCompareAndSendConfiguration(t *testing.T) { + testValue := []byte("test") + testErr := errors.New("test error") + decryptFn := func(_ context.Context, payload []byte) ([]byte, error) { + if string(payload) == string(testValue) { + return testValue, nil + } + return nil, testErr + } + + var got string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("content-type", "application/json") + + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, r.Body.Close()) + got = string(b) + + _, err = w.Write([]byte(`{"status": "success"}`)) + require.NoError(t, err) + })) + + fstore := notifier.NewFileStore(1, ngfakes.NewFakeKVStore(t), "") + m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) + cfg := AlertmanagerConfig{ + OrgID: 1, + TenantID: "test", + URL: server.URL, + } + am, err := NewAlertmanager(cfg, + fstore, + decryptFn, + m, + ) + require.NoError(t, err) + + tests := []struct { + name string + config string + expCfg *client.UserGrafanaConfig + expErr string + }{ + { + "invalid config", + "{}", + nil, + "unable to parse Alertmanager configuration: no route provided in config", + }, + { + "invalid base-64 in key", + strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, `"password":"!"`, 1), + nil, + "failed to decode value for key 'password': illegal base64 data at input byte 0", + }, + { + "decrypt error", + testGrafanaConfigWithSecret, + nil, + fmt.Sprintf("failed to decrypt value for key 'password': %s", testErr.Error()), + }, + { + "no error", + strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, fmt.Sprintf("%q:%q", "password", base64.StdEncoding.EncodeToString(testValue)), 1), + &client.UserGrafanaConfig{ + GrafanaAlertmanagerConfig: testGrafanaConfigWithSecret, + }, + "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + cfg := ngmodels.AlertConfiguration{ + AlertmanagerConfiguration: test.config, + } + err = am.CompareAndSendConfiguration(context.Background(), &cfg) + if test.expErr == "" { + require.NoError(tt, err) + rawCfg, err := json.Marshal(test.expCfg) + require.NoError(tt, err) + require.JSONEq(tt, string(rawCfg), got) + return + } + require.Equal(tt, test.expErr, err.Error()) + }) + } +} + func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -162,7 +296,7 @@ func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) { silences := []byte("test-silences") nflog := []byte("test-notifications") - store := fakes.NewFakeKVStore(t) + store := ngfakes.NewFakeKVStore(t) fstore := notifier.NewFileStore(cfg.OrgID, store, "") ctx := context.Background() @@ -179,8 +313,9 @@ func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) { require.NoError(t, err) encodedFullState := base64.StdEncoding.EncodeToString(fullState) + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, fstore, m) + am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, m) require.NoError(t, err) // We should have no configuration or state at first. @@ -264,8 +399,10 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) { TenantID: tenantID, BasicAuthPassword: password, } + + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, nil, m) + am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, m) require.NoError(t, err) // We should have no silences at first. @@ -345,8 +482,10 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) { TenantID: tenantID, BasicAuthPassword: password, } + + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, nil, m) + am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, m) require.NoError(t, err) // Wait until the Alertmanager is ready to send alerts. @@ -412,8 +551,9 @@ func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) { BasicAuthPassword: password, } + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) - am, err := NewAlertmanager(cfg, nil, m) + am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, m) require.NoError(t, err) // We should start with the default config.