Alerting: Use new receiver models for encrypt/decrypt in remote AM (#107042)

Several niche bugs have surfaced as a result of the decrypt code Grafana uses in receivers API being different than what is used to decrypt secrets before sending to remote AM. Example:
- Dingding notifier not abiding by new Patching added to local AM, thus causing missing url errors.

* noop refactor to simplify decryptConfiguration

* Move compat function package

* Use new receiver models to encrypt/decrypt in remote AM

pkg/services/ngalert/notifier/compat.go

@@ -2,102 +2,13 @@ package notifier
 
 import (
 	"encoding/json"
-	"fmt"
 
 	alertingNotify "github.com/grafana/alerting/notify"
 
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
-	"github.com/grafana/grafana/pkg/services/ngalert/notifier/legacy_storage"
 )
 
-func PostableApiReceiversToReceivers(postables []*apimodels.PostableApiReceiver, storedProvenances map[string]models.Provenance) ([]*models.Receiver, error) {
-	receivers := make([]*models.Receiver, 0, len(postables))
-	for _, postable := range postables {
-		r, err := PostableApiReceiverToReceiver(postable, getReceiverProvenance(storedProvenances, postable))
-		if err != nil {
-			return nil, err
-		}
-		receivers = append(receivers, r)
-	}
-	return receivers, nil
-}
-
-func PostableApiReceiverToReceiver(postable *apimodels.PostableApiReceiver, provenance models.Provenance) (*models.Receiver, error) {
-	integrations, err := PostableGrafanaReceiversToIntegrations(postable.GrafanaManagedReceivers)
-	if err != nil {
-		return nil, err
-	}
-	r := &models.Receiver{
-		UID:          legacy_storage.NameToUid(postable.GetName()), // TODO replace with stable UID.
-		Name:         postable.GetName(),
-		Integrations: integrations,
-		Provenance:   provenance,
-	}
-	r.Version = r.Fingerprint()
-	return r, nil
-}
-
-func PostableGrafanaReceiversToIntegrations(postables []*apimodels.PostableGrafanaReceiver) ([]*models.Integration, error) {
-	integrations := make([]*models.Integration, 0, len(postables))
-	for _, cfg := range postables {
-		integration, err := PostableGrafanaReceiverToIntegration(cfg)
-		if err != nil {
-			return nil, err
-		}
-		integrations = append(integrations, integration)
-	}
-
-	return integrations, nil
-}
-
-func PostableGrafanaReceiverToIntegration(p *apimodels.PostableGrafanaReceiver) (*models.Integration, error) {
-	config, err := models.IntegrationConfigFromType(p.Type)
-	if err != nil {
-		return nil, err
-	}
-	integration := &models.Integration{
-		UID:                   p.UID,
-		Name:                  p.Name,
-		Config:                config,
-		DisableResolveMessage: p.DisableResolveMessage,
-		Settings:              make(map[string]any, len(p.Settings)),
-		SecureSettings:        make(map[string]string, len(p.SecureSettings)),
-	}
-
-	if p.Settings != nil {
-		if err := json.Unmarshal(p.Settings, &integration.Settings); err != nil {
-			return nil, fmt.Errorf("integration '%s' of receiver '%s' has settings that cannot be parsed as JSON: %w", integration.Config.Type, p.Name, err)
-		}
-	}
-
-	for k, v := range p.SecureSettings {
-		if v != "" {
-			integration.SecureSettings[k] = v
-		}
-	}
-
-	return integration, nil
-}
-
-// getReceiverProvenance determines the provenance of a definitions.PostableApiReceiver based on the provenance of its integrations.
-func getReceiverProvenance(storedProvenances map[string]models.Provenance, r *apimodels.PostableApiReceiver) models.Provenance {
-	if len(r.GrafanaManagedReceivers) == 0 {
-		return models.ProvenanceNone
-	}
-
-	// Current provisioning works on the integration level, so we need some way to determine the provenance of the
-	// entire receiver. All integrations in a receiver should have the same provenance, but we don't want to rely on
-	// this assumption in case the first provenance is None and a later one is not. To this end, we return the first
-	// non-zero provenance we find.
-	for _, contactPoint := range r.GrafanaManagedReceivers {
-		if p, exists := storedProvenances[contactPoint.UID]; exists && p != models.ProvenanceNone {
-			return p
-		}
-	}
-	return models.ProvenanceNone
-}
-
 func PostableGrafanaReceiverToGrafanaIntegrationConfig(p *apimodels.PostableGrafanaReceiver) *alertingNotify.GrafanaIntegrationConfig {
 	return &alertingNotify.GrafanaIntegrationConfig{
 		UID:                   p.UID,

pkg/services/ngalert/notifier/legacy_storage/compat.go

@@ -3,6 +3,7 @@ package legacy_storage
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"maps"
 
 	alertingNotify "github.com/grafana/alerting/notify"
@@ -63,3 +64,90 @@ func ReceiverToPostableApiReceiver(r *models.Receiver) (*apimodels.PostableApiRe
 		PostableGrafanaReceivers: integrations,
 	}, nil
 }
+
+func PostableApiReceiversToReceivers(postables []*apimodels.PostableApiReceiver, storedProvenances map[string]models.Provenance) ([]*models.Receiver, error) {
+	receivers := make([]*models.Receiver, 0, len(postables))
+	for _, postable := range postables {
+		r, err := PostableApiReceiverToReceiver(postable, GetReceiverProvenance(storedProvenances, postable))
+		if err != nil {
+			return nil, err
+		}
+		receivers = append(receivers, r)
+	}
+	return receivers, nil
+}
+
+func PostableApiReceiverToReceiver(postable *apimodels.PostableApiReceiver, provenance models.Provenance) (*models.Receiver, error) {
+	integrations, err := PostableGrafanaReceiversToIntegrations(postable.GrafanaManagedReceivers)
+	if err != nil {
+		return nil, err
+	}
+	r := &models.Receiver{
+		UID:          NameToUid(postable.GetName()), // TODO replace with stable UID.
+		Name:         postable.GetName(),
+		Integrations: integrations,
+		Provenance:   provenance,
+	}
+	r.Version = r.Fingerprint()
+	return r, nil
+}
+
+// GetReceiverProvenance determines the provenance of a definitions.PostableApiReceiver based on the provenance of its integrations.
+func GetReceiverProvenance(storedProvenances map[string]models.Provenance, r *apimodels.PostableApiReceiver) models.Provenance {
+	if len(r.GrafanaManagedReceivers) == 0 {
+		return models.ProvenanceNone
+	}
+
+	// Current provisioning works on the integration level, so we need some way to determine the provenance of the
+	// entire receiver. All integrations in a receiver should have the same provenance, but we don't want to rely on
+	// this assumption in case the first provenance is None and a later one is not. To this end, we return the first
+	// non-zero provenance we find.
+	for _, contactPoint := range r.GrafanaManagedReceivers {
+		if p, exists := storedProvenances[contactPoint.UID]; exists && p != models.ProvenanceNone {
+			return p
+		}
+	}
+	return models.ProvenanceNone
+}
+
+func PostableGrafanaReceiversToIntegrations(postables []*apimodels.PostableGrafanaReceiver) ([]*models.Integration, error) {
+	integrations := make([]*models.Integration, 0, len(postables))
+	for _, cfg := range postables {
+		integration, err := PostableGrafanaReceiverToIntegration(cfg)
+		if err != nil {
+			return nil, err
+		}
+		integrations = append(integrations, integration)
+	}
+
+	return integrations, nil
+}
+
+func PostableGrafanaReceiverToIntegration(p *apimodels.PostableGrafanaReceiver) (*models.Integration, error) {
+	config, err := models.IntegrationConfigFromType(p.Type)
+	if err != nil {
+		return nil, err
+	}
+	integration := &models.Integration{
+		UID:                   p.UID,
+		Name:                  p.Name,
+		Config:                config,
+		DisableResolveMessage: p.DisableResolveMessage,
+		Settings:              make(map[string]any, len(p.Settings)),
+		SecureSettings:        make(map[string]string, len(p.SecureSettings)),
+	}
+
+	if p.Settings != nil {
+		if err := json.Unmarshal(p.Settings, &integration.Settings); err != nil {
+			return nil, fmt.Errorf("integration '%s' of receiver '%s' has settings that cannot be parsed as JSON: %w", integration.Config.Type, p.Name, err)
+		}
+	}
+
+	for k, v := range p.SecureSettings {
+		if v != "" {
+			integration.SecureSettings[k] = v
+		}
+	}
+
+	return integration, nil
+}

pkg/services/ngalert/notifier/legacy_storage/receivers.go

@@ -101,6 +101,52 @@ func (rev *ConfigRevision) GetReceivers(uids []string) []*definitions.PostableAp
 	return receivers
 }
 
+func DecryptedReceivers(receivers []*definitions.PostableApiReceiver, decryptFn models.DecryptFn) ([]*definitions.PostableApiReceiver, error) {
+	decrypted := make([]*definitions.PostableApiReceiver, len(receivers))
+	for i, r := range receivers {
+		// We don't care about the provenance here, so we pass ProvenanceNone.
+		rcv, err := PostableApiReceiverToReceiver(r, models.ProvenanceNone)
+		if err != nil {
+			return nil, err
+		}
+
+		err = rcv.Decrypt(decryptFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decrypt receiver %q: %w", rcv.Name, err)
+		}
+
+		postable, err := ReceiverToPostableApiReceiver(rcv)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert Receiver %q to APIReceiver: %w", rcv.Name, err)
+		}
+		decrypted[i] = postable
+	}
+	return decrypted, nil
+}
+
+func EncryptedReceivers(receivers []*definitions.PostableApiReceiver, encryptFn models.EncryptFn) ([]*definitions.PostableApiReceiver, error) {
+	encrypted := make([]*definitions.PostableApiReceiver, len(receivers))
+	for i, r := range receivers {
+		// We don't care about the provenance here, so we pass ProvenanceNone.
+		rcv, err := PostableApiReceiverToReceiver(r, models.ProvenanceNone)
+		if err != nil {
+			return nil, err
+		}
+
+		err = rcv.Encrypt(encryptFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decrypt receiver %q: %w", rcv.Name, err)
+		}
+
+		postable, err := ReceiverToPostableApiReceiver(rcv)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert Receiver %q to APIReceiver: %w", rcv.Name, err)
+		}
+		encrypted[i] = postable
+	}
+	return encrypted, nil
+}
+
 // RenameReceiverInRoutes renames all references to a receiver in routes. Returns number of routes that were updated
 func (rev *ConfigRevision) RenameReceiverInRoutes(oldName, newName string) int {
 	return RenameReceiverInRoute(oldName, newName, rev.Config.AlertmanagerConfig.Route)

pkg/services/ngalert/notifier/receiver_svc.go

@@ -146,7 +146,7 @@ func (rs *ReceiverService) GetReceiver(ctx context.Context, q models.GetReceiver
 	if err != nil {
 		return nil, err
 	}
-	rcv, err := PostableApiReceiverToReceiver(postable, getReceiverProvenance(storedProvenances, postable))
+	rcv, err := legacy_storage.PostableApiReceiverToReceiver(postable, legacy_storage.GetReceiverProvenance(storedProvenances, postable))
 	if err != nil {
 		return nil, err
 	}
@@ -206,7 +206,7 @@ func (rs *ReceiverService) GetReceivers(ctx context.Context, q models.GetReceive
 	if err != nil {
 		return nil, err
 	}
-	receivers, err := PostableApiReceiversToReceivers(postables, storedProvenances)
+	receivers, err := legacy_storage.PostableApiReceiversToReceivers(postables, storedProvenances)
 	if err != nil {
 		return nil, err
 	}
@@ -280,7 +280,7 @@ func (rs *ReceiverService) ListReceivers(ctx context.Context, q models.ListRecei
 	if err != nil {
 		return nil, err
 	}
-	receivers, err := PostableApiReceiversToReceivers(postables, storedProvenances)
+	receivers, err := legacy_storage.PostableApiReceiversToReceivers(postables, storedProvenances)
 	if err != nil {
 		return nil, err
 	}
@@ -337,7 +337,7 @@ func (rs *ReceiverService) DeleteReceiver(ctx context.Context, uid string, calle
 	if err != nil {
 		return err
 	}
-	existing, err := PostableApiReceiverToReceiver(postable, getReceiverProvenance(storedProvenances, postable))
+	existing, err := legacy_storage.PostableApiReceiverToReceiver(postable, legacy_storage.GetReceiverProvenance(storedProvenances, postable))
 	if err != nil {
 		return err
 	}
@@ -439,7 +439,7 @@ func (rs *ReceiverService) CreateReceiver(ctx context.Context, r *models.Receive
 		return nil, err
 	}
 
-	result, err = PostableApiReceiverToReceiver(created, createdReceiver.Provenance)
+	result, err = legacy_storage.PostableApiReceiverToReceiver(created, createdReceiver.Provenance)
 	if err != nil {
 		return nil, err
 	}
@@ -480,7 +480,7 @@ func (rs *ReceiverService) UpdateReceiver(ctx context.Context, r *models.Receive
 	if err != nil {
 		return nil, err
 	}
-	existing, err := PostableApiReceiverToReceiver(postable, getReceiverProvenance(storedProvenances, postable))
+	existing, err := legacy_storage.PostableApiReceiverToReceiver(postable, legacy_storage.GetReceiverProvenance(storedProvenances, postable))
 	if err != nil {
 		return nil, err
 	}
@@ -567,7 +567,7 @@ func (rs *ReceiverService) UpdateReceiver(ctx context.Context, r *models.Receive
 		return nil, err
 	}
 
-	result, err := PostableApiReceiverToReceiver(updated, updatedReceiver.Provenance)
+	result, err := legacy_storage.PostableApiReceiverToReceiver(updated, updatedReceiver.Provenance)
 	if err != nil {
 		return nil, err
 	}

pkg/services/ngalert/notifier/receiver_svc_test.go

@@ -728,7 +728,7 @@ func TestReceiverService_Update(t *testing.T) {
 				result, err := revision.CreateReceiver(tc.existing)
 				require.NoError(t, err)
 
-				created, err := PostableApiReceiverToReceiver(result, tc.existing.Provenance)
+				created, err := legacy_storage.PostableApiReceiverToReceiver(result, tc.existing.Provenance)
 				require.NoError(t, err)
 				err = sut.cfgStore.Save(context.Background(), revision, tc.user.GetOrgID())
 				require.NoError(t, err)

pkg/services/ngalert/remote/alertmanager.go

@@ -30,6 +30,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier/legacy_storage"
 	remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/sender"
 )
@@ -280,22 +281,22 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 	if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
 		return err
 	}
-	rawDecrypted, configHash, err := am.decryptConfiguration(ctx, c)
+	decryptedCfg, err := am.decryptConfiguration(ctx, c)
 	if err != nil {
 		return err
 	}
+	rawDecrypted, err := json.Marshal(decryptedCfg)
+	if err != nil {
+		return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
+	}
+	configHash := md5.Sum(rawDecrypted)
 
 	// Send the configuration only if we need to.
 	if !am.shouldSendConfig(ctx, configHash) {
 		return nil
 	}
 
-	decrypted, err := notifier.Load(rawDecrypted)
-	if err != nil {
-		return err
-	}
-
-	return am.sendConfiguration(ctx, decrypted, config.ConfigurationHash, config.CreatedAt, am.isDefaultConfiguration(configHash))
+	return am.sendConfiguration(ctx, decryptedCfg, config.ConfigurationHash, config.CreatedAt, am.isDefaultConfiguration(configHash))
 }
 
 func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) bool {
@@ -305,38 +306,39 @@ func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) bool {
 // decryptConfiguration creates a copy of the configuration, decrypts it, and returns the decrypted configuration alongside its hash.
 // Should not be used outside of this package and the specific use case of decrypting the configuration before sending
 // it to the remote Alertmanager.
-func (am *Alertmanager) decryptConfiguration(ctx context.Context, cfg *apimodels.PostableUserConfig) ([]byte, [16]byte, error) {
-	fn := func(payload []byte) ([]byte, error) {
-		return am.decrypt(ctx, payload)
-	}
-
+func (am *Alertmanager) decryptConfiguration(ctx context.Context, cfg *apimodels.PostableUserConfig) (*apimodels.PostableUserConfig, error) {
 	// Create a copy of the configuration to avoid modifying the original
 	cfgCopy := &apimodels.PostableUserConfig{}
 	rawCfg, err := json.Marshal(cfg)
 	if err != nil {
-		return nil, [16]byte{}, fmt.Errorf("unable to marshal original configuration: %w", err)
+		return nil, fmt.Errorf("unable to marshal original configuration: %w", err)
 	}
 	if err := json.Unmarshal(rawCfg, cfgCopy); err != nil {
-		return nil, [16]byte{}, fmt.Errorf("unable to unmarshal original configuration: %w", err)
+		return nil, fmt.Errorf("unable to unmarshal original configuration: %w", err)
 	}
 
-	// Iterate through receivers and decrypt secure settings on the copy
-	for _, rcv := range cfgCopy.AlertmanagerConfig.Receivers {
-		for _, gmr := range rcv.GrafanaManagedReceivers {
-			decrypted, err := gmr.DecryptSecureSettings(fn)
-			if err != nil {
-				return nil, [16]byte{}, fmt.Errorf("unable to decrypt settings on receiver %q (uid: %q): %w", gmr.Name, gmr.UID, err)
-			}
-			gmr.SecureSettings = decrypted
-		}
-	}
-
-	rawDecrypted, err := json.Marshal(cfgCopy)
+	// Decrypt the receivers in the configuration.
+	decryptedReceivers, err := legacy_storage.DecryptedReceivers(cfgCopy.AlertmanagerConfig.Receivers, decrypter(ctx, am.decrypt))
 	if err != nil {
-		return nil, [16]byte{}, fmt.Errorf("unable to marshal decrypted configuration: %w", err)
+		return nil, fmt.Errorf("unable to decrypt receivers: %w", err)
 	}
+	cfgCopy.AlertmanagerConfig.Receivers = decryptedReceivers
 
-	return rawDecrypted, md5.Sum(rawDecrypted), nil
+	return cfgCopy, nil
+}
+
+func decrypter(ctx context.Context, decryptFn DecryptFn) models.DecryptFn {
+	return func(value string) (string, error) {
+		decoded, err := base64.StdEncoding.DecodeString(value)
+		if err != nil {
+			return "", err
+		}
+		decrypted, err := decryptFn(ctx, decoded)
+		if err != nil {
+			return "", err
+		}
+		return string(decrypted), nil
+	}
 }
 
 func (am *Alertmanager) sendConfiguration(ctx context.Context, decrypted *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
@@ -388,17 +390,12 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 	if err := am.autogenFn(ctx, am.log, am.orgID, &cfg.AlertmanagerConfig, false); err != nil {
 		return err
 	}
-	rawDecrypted, _, err := am.decryptConfiguration(ctx, cfg)
+	decryptedCfg, err := am.decryptConfiguration(ctx, cfg)
 	if err != nil {
 		return err
 	}
 
-	decrypted, err := notifier.Load(rawDecrypted)
-	if err != nil {
-		return err
-	}
-
-	return am.sendConfiguration(ctx, decrypted, hash, time.Now().Unix(), false)
+	return am.sendConfiguration(ctx, decryptedCfg, hash, time.Now().Unix(), false)
 }
 
 // SaveAndApplyDefaultConfig sends the default Grafana Alertmanager configuration to the remote Alertmanager.
@@ -412,19 +409,14 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
 	if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
 		return err
 	}
-	rawDecrypted, _, err := am.decryptConfiguration(ctx, c)
-	if err != nil {
-		return err
-	}
-
-	decrypted, err := notifier.Load(rawDecrypted)
+	decryptedCfg, err := am.decryptConfiguration(ctx, c)
 	if err != nil {
 		return err
 	}
 
 	return am.sendConfiguration(
 		ctx,
-		decrypted,
+		decryptedCfg,
 		am.defaultConfigHash,
 		time.Now().Unix(),
 		true,
@@ -581,34 +573,14 @@ func (am *Alertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver,
 }
 
 func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error) {
-	fn := func(payload []byte) ([]byte, error) {
-		return am.decrypt(ctx, payload)
+	decryptedReceivers, err := legacy_storage.DecryptedReceivers(c.Receivers, decrypter(ctx, am.decrypt))
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to decrypt receivers: %w", err)
 	}
 
-	receivers := make([]*alertingNotify.APIReceiver, 0, len(c.Receivers))
-	for _, r := range c.Receivers {
-		integrations := make([]*alertingNotify.GrafanaIntegrationConfig, 0, len(r.GrafanaManagedReceivers))
-		for _, gr := range r.GrafanaManagedReceivers {
-			decrypted, err := gr.DecryptSecureSettings(fn)
-			if err != nil {
-				return nil, 0, err
-			}
-
-			integrations = append(integrations, &alertingNotify.GrafanaIntegrationConfig{
-				UID:                   gr.UID,
-				Name:                  gr.Name,
-				Type:                  gr.Type,
-				DisableResolveMessage: gr.DisableResolveMessage,
-				Settings:              json.RawMessage(gr.Settings),
-				SecureSettings:        decrypted,
-			})
-		}
-		receivers = append(receivers, &alertingNotify.APIReceiver{
-			ConfigReceiver: r.Receiver,
-			GrafanaIntegrations: alertingNotify.GrafanaIntegrations{
-				Integrations: integrations,
-			},
-		})
+	apiReceivers := make([]*alertingNotify.APIReceiver, 0, len(c.Receivers))
+	for _, r := range decryptedReceivers {
+		apiReceivers = append(apiReceivers, notifier.PostableApiReceiverToApiReceiver(r))
 	}
 	var alert *alertingNotify.TestReceiversConfigAlertParams
 	if c.Alert != nil {
@@ -617,7 +589,7 @@ func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestRecei
 
 	return am.mimirClient.TestReceivers(ctx, alertingNotify.TestReceiversConfigBodyParams{
 		Alert:     alert,
-		Receivers: receivers,
+		Receivers: apiReceivers,
 	})
 }
 

pkg/services/ngalert/remote/alertmanager_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier/legacy_storage"
 	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
 	"github.com/grafana/grafana/pkg/services/secrets"
@@ -44,12 +45,8 @@ import (
 )
 
 var (
-	testPasswordBase64   = base64.StdEncoding.EncodeToString([]byte(testPassword))
 	defaultGrafanaConfig = setting.GetAlertmanagerDefaultConfiguration()
 	errTest              = errors.New("test")
-
-	// Valid Grafana Alertmanager configuration with secret in base64.
-	testGrafanaConfigWithEncryptedSecret = fmt.Sprintf(`{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"%s"}}]}]}}`, testPasswordBase64)
 )
 
 const (
@@ -57,7 +54,7 @@ const (
 
 	// Valid Grafana Alertmanager configurations.
 	testGrafanaConfig                               = `{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"}}]}]}}`
-	testGrafanaConfigWithSecret                     = `{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"test"}}]}]}}`
+	testGrafanaConfigWithSecret                     = `{"template_files":{},"alertmanager_config":{"time_intervals":[{"name":"weekends","time_intervals":[{"weekdays":["saturday","sunday"],"location":"Africa/Accra"}]}],"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test","password":"test"}}]}]}}`
 	testGrafanaDefaultConfigWithDifferentFieldOrder = `{"alertmanager_config":{"route":{"group_by":["alertname","grafana_folder"],"receiver":"grafana-default-email"},"receivers":[{"grafana_managed_receiver_configs":[{"uid":"","name":"email receiver","type":"email","settings":{"addresses":"<example@email.com>"}}],"name":"grafana-default-email"}]}}`
 
 	// Valid Alertmanager state base64 encoded.
@@ -167,9 +164,14 @@ func TestApplyConfig(t *testing.T) {
 	var c apimodels.PostableUserConfig
 	require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &c))
 	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
-	err := notifier.EncryptReceiverConfigs(c.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) {
-		return secretsService.Encrypt(ctx, payload, secrets.WithoutScope())
+	encryptedReceivers, err := legacy_storage.EncryptedReceivers(c.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
+		encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
+		if err != nil {
+			return "", err
+		}
+		return base64.StdEncoding.EncodeToString(encrypted), nil
 	})
+	c.AlertmanagerConfig.Receivers = encryptedReceivers
 	require.NoError(t, err)
 
 	// The encrypted configuration should be different than the one we will send.
@@ -289,15 +291,7 @@ func TestApplyConfig(t *testing.T) {
 
 func TestCompareAndSendConfiguration(t *testing.T) {
 	const tenantID = "test"
-	cfgWithSecret, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
-	require.NoError(t, err)
-	testValue := []byte("test")
-	decryptFn := func(_ context.Context, payload []byte) ([]byte, error) {
-		if string(payload) == string(testValue) {
-			return testValue, nil
-		}
-		return nil, errTest
-	}
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
 
 	var got string
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -333,62 +327,90 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 		return nil
 	}
 
+	// Create a config with correctly encrypted and encoded secrets.
+	var inputCfg apimodels.PostableUserConfig
+	require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &inputCfg))
+	encryptedReceivers, err := legacy_storage.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
+		encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
+		if err != nil {
+			return "", err
+		}
+		return base64.StdEncoding.EncodeToString(encrypted), nil
+	})
+	inputCfg.AlertmanagerConfig.Receivers = encryptedReceivers
+	require.NoError(t, err)
+	testGrafanaConfigWithEncryptedSecret, err := json.Marshal(inputCfg)
+	require.NoError(t, err)
+
+	// Created a config with invalid base64 encoding in the secret.
+	inputCfg.AlertmanagerConfig.Receivers[0].PostableGrafanaReceivers.GrafanaManagedReceivers[0].SecureSettings["password"] = "!"
+	testGrafanaConfigWithBadEncoding, err := json.Marshal(inputCfg)
+	require.NoError(t, err)
+
+	// Create a config with a valid base64 encoding but an invalid encryption.
+	inputCfg.AlertmanagerConfig.Receivers[0].PostableGrafanaReceivers.GrafanaManagedReceivers[0].SecureSettings["password"] = base64.StdEncoding.EncodeToString([]byte("test"))
+	testGrafanaConfigWithBadEncryption, err := json.Marshal(inputCfg)
+	require.NoError(t, err)
+
+	cfgWithDecryptedSecret, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
+	require.NoError(t, err)
+
 	cfgWithAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
 	require.NoError(t, err)
 	require.NoError(t, testAutogenFn(nil, nil, 0, &cfgWithAutogenRoutes.AlertmanagerConfig, false))
 
 	tests := []struct {
-		name      string
-		config    string
-		autogenFn AutogenFn
-		expCfg    *client.UserGrafanaConfig
-		expErr    string
+		name           string
+		config         string
+		autogenFn      AutogenFn
+		expCfg         *client.UserGrafanaConfig
+		expErrContains []string
 	}{
 		{
 			"invalid config",
 			"{}",
 			NoopAutogenFn,
 			nil,
-			"unable to parse Alertmanager configuration: no route provided in config",
+			[]string{"no route provided in config"},
 		},
 		{
 			"invalid base-64 in key",
-			strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, `"password":"!"`, 1),
+			string(testGrafanaConfigWithBadEncoding),
 			NoopAutogenFn,
 			nil,
-			`unable to decrypt settings on receiver "WH" (uid: "dde6ntuob69dtf"): failed to decode value for key 'password': illegal base64 data at input byte 0`,
+			[]string{`"grafana-default-email"`, "dde6ntuob69dtf", "password", "illegal base64 data at input byte 0"},
 		},
 		{
 			"decrypt error",
-			testGrafanaConfigWithSecret,
+			string(testGrafanaConfigWithBadEncryption),
 			NoopAutogenFn,
 			nil,
-			fmt.Sprintf(`unable to decrypt settings on receiver "WH" (uid: "dde6ntuob69dtf"): failed to decrypt value for key 'password': %s`, errTest.Error()),
+			[]string{`"grafana-default-email"`, "dde6ntuob69dtf", "password", "unable to compute salt"},
 		},
 		{
 			"error from autogen function",
-			strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, fmt.Sprintf("%q:%q", "password", base64.StdEncoding.EncodeToString(testValue)), 1),
+			string(testGrafanaConfigWithEncryptedSecret),
 			errAutogenFn,
 			nil,
-			errTest.Error(),
+			[]string{errTest.Error()},
 		},
 		{
 			"no error",
-			strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, fmt.Sprintf("%q:%q", "password", base64.StdEncoding.EncodeToString(testValue)), 1),
+			string(testGrafanaConfigWithEncryptedSecret),
 			NoopAutogenFn,
 			&client.UserGrafanaConfig{
-				GrafanaAlertmanagerConfig: cfgWithSecret,
+				GrafanaAlertmanagerConfig: cfgWithDecryptedSecret,
 			},
-			"",
+			nil,
 		},
 		{
 			"no error, with auto-generated routes",
-			strings.Replace(testGrafanaConfigWithSecret, `"password":"test"`, fmt.Sprintf("%q:%q", "password", base64.StdEncoding.EncodeToString(testValue)), 1),
+			string(testGrafanaConfigWithEncryptedSecret),
 			testAutogenFn,
 			&client.UserGrafanaConfig{
 				GrafanaAlertmanagerConfig: cfgWithAutogenRoutes,
 			},
-			"",
+			nil,
 		},
 	}
 
@@ -398,7 +420,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 			am, err := NewAlertmanager(ctx,
 				cfg,
 				fstore,
-				decryptFn,
+				secretsService.Decrypt,
 				NoopAutogenFn,
 				m,
 				tracing.InitializeTracerForTest(),
@@ -413,29 +435,23 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 				AlertmanagerConfiguration: test.config,
 			}
 			err = am.CompareAndSendConfiguration(ctx, &cfg)
-			if test.expErr == "" {
+			if len(test.expErrContains) == 0 {
 				require.NoError(tt, err)
 				rawCfg, err := json.Marshal(test.expCfg)
 				require.NoError(tt, err)
 				require.JSONEq(tt, string(rawCfg), got)
 				return
 			}
-			require.Equal(tt, test.expErr, err.Error())
+			for _, expErr := range test.expErrContains {
+				require.ErrorContains(tt, err, expErr)
+			}
 		})
 	}
 }
 
 func Test_TestReceiversDecryptsSecureSettings(t *testing.T) {
 	const tenantID = "test"
-	const testKey = "test-key"
-	const testValue = "test-value"
-	decryptFn := func(_ context.Context, payload []byte) ([]byte, error) {
-		if string(payload) == testValue {
-			return []byte(testValue), nil
-		}
-		return nil, errTest
-	}
-
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
 	var got apimodels.TestReceiversConfigBodyParams
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader))
@@ -459,33 +475,40 @@ func Test_TestReceiversDecryptsSecureSettings(t *testing.T) {
 	am, err := NewAlertmanager(context.Background(),
 		cfg,
 		fstore,
-		decryptFn,
+		secretsService.Decrypt,
 		NoopAutogenFn,
 		m,
 		tracing.InitializeTracerForTest(),
 	)
-
 	require.NoError(t, err)
+
+	var inputCfg apimodels.PostableUserConfig
+	require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &inputCfg))
+	encryptedReceivers, err := legacy_storage.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
+		encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
+		if err != nil {
+			return "", err
+		}
+		return base64.StdEncoding.EncodeToString(encrypted), nil
+	})
+	inputCfg.AlertmanagerConfig.Receivers = encryptedReceivers
+	require.NoError(t, err)
+
 	params := apimodels.TestReceiversConfigBodyParams{
-		Alert: &apimodels.TestReceiversConfigAlertParams{},
-		Receivers: []*definition.PostableApiReceiver{
-			{
-				PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{
-					GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{
-						{
-							SecureSettings: map[string]string{
-								testKey: base64.StdEncoding.EncodeToString([]byte(testValue)),
-							},
-						},
-					},
-				},
-			},
-		},
+		Alert:     &apimodels.TestReceiversConfigAlertParams{},
+		Receivers: inputCfg.AlertmanagerConfig.Receivers,
 	}
 
 	_, _, err = am.TestReceivers(context.Background(), params)
 	require.NoError(t, err)
-	require.Equal(t, map[string]string{testKey: testValue}, got.Receivers[0].PostableGrafanaReceivers.GrafanaManagedReceivers[0].SecureSettings)
+
+	expectedSettings, err := json.Marshal(map[string]any{
+		"url":      "http://localhost:8080",
+		"username": "test",
+		"password": testPassword,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, expectedSettings, got.Receivers[0].PostableGrafanaReceivers.GrafanaManagedReceivers[0].Settings)
 }
 
 func Test_isDefaultConfiguration(t *testing.T) {
@@ -533,44 +556,6 @@ func Test_isDefaultConfiguration(t *testing.T) {
 	}
 }
 
-func TestDecryptConfiguration(t *testing.T) {
-	t.Run("should not modify the original config", func(t *testing.T) {
-		var inputCfg apimodels.PostableUserConfig
-		require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithEncryptedSecret), &inputCfg))
-
-		decryptFn := func(_ context.Context, payload []byte) ([]byte, error) {
-			if string(payload) == testPassword {
-				return []byte(testPassword), nil
-			}
-			return nil, fmt.Errorf("incorrect payload")
-		}
-
-		am := &Alertmanager{
-			decrypt: decryptFn,
-		}
-
-		rawDecrypted, _, err := am.decryptConfiguration(context.Background(), &inputCfg)
-		require.NoError(t, err)
-
-		currentJSON, err := json.Marshal(inputCfg)
-		require.NoError(t, err)
-		require.JSONEq(t, testGrafanaConfigWithEncryptedSecret, string(currentJSON), "Original configuration should not be modified")
-
-		var decryptedCfg apimodels.PostableUserConfig
-		require.NoError(t, json.Unmarshal(rawDecrypted, &decryptedCfg))
-
-		found := false
-		for _, rcv := range decryptedCfg.AlertmanagerConfig.Receivers {
-			for _, gmr := range rcv.GrafanaManagedReceivers {
-				if gmr.Type == "webhook" && gmr.SecureSettings["password"] == testPassword {
-					found = true
-				}
-			}
-		}
-		require.True(t, found, "Decrypted configuration should contain decrypted password")
-	})
-}
-
 func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
@@ -684,9 +669,14 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 	{
 		postableCfg, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
 		require.NoError(t, err)
-		err = notifier.EncryptReceiverConfigs(postableCfg.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) {
-			return secretsService.Encrypt(ctx, payload, secrets.WithoutScope())
+		encryptedReceivers, err := legacy_storage.EncryptedReceivers(postableCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
+			encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
+			if err != nil {
+				return "", err
+			}
+			return base64.StdEncoding.EncodeToString(encrypted), nil
 		})
+		postableCfg.AlertmanagerConfig.Receivers = encryptedReceivers
 		require.NoError(t, err)
 
 		// The encrypted configuration should be different than the one we will send.
@@ -698,6 +688,11 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 		require.NoError(t, err)
 		require.NoError(t, am.SaveAndApplyConfig(ctx, postableCfg))
 
+		// Check that the original configuration is not modified (decrypted).
+		currentJSON, err := json.Marshal(postableCfg)
+		require.NoError(t, err)
+		require.JSONEq(t, string(encryptedConfig), string(currentJSON), "Original configuration should not be modified")
+
 		// Check that the configuration was uploaded to the remote Alertmanager.
 		config, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx)
 		require.NoError(t, err)