Alerting: Send merged configuration to the remote alertmanager (#107004)

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -1051,21 +1051,9 @@ func (c *GettableApiAlertingConfig) UnmarshalYAML(value *yaml.Node) error {
 func (c *GettableApiAlertingConfig) validate() error {
 	receivers := make(map[string]struct{}, len(c.Receivers))
 
-	var hasGrafReceivers, hasAMReceivers bool
-	for _, r := range c.Receivers {
-		receivers[r.Name] = struct{}{}
-		switch r.Type() {
-		case GrafanaReceiverType:
-			hasGrafReceivers = true
-		case AlertmanagerReceiverType:
-			hasAMReceivers = true
-		default:
-			continue
-		}
-	}
-
-	if hasGrafReceivers && hasAMReceivers {
-		return fmt.Errorf("cannot mix Alertmanager & Grafana receiver types")
+	// Populate the receivers map with defined receiver names
+	for _, receiver := range c.Receivers {
+		receivers[receiver.Name] = struct{}{}
 	}
 
 	for _, receiver := range AllReceivers(c.Route.AsAMRoute()) {

pkg/services/ngalert/notifier/crypto.go

@@ -256,9 +256,11 @@ func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *de
 
 func (c *alertmanagerCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
 	for i := range config.ExtraConfigs {
+		// Check if the config is encrypted by trying to base64 decode it
 		encryptedValue, err := base64.StdEncoding.DecodeString(config.ExtraConfigs[i].AlertmanagerConfig)
 		if err != nil {
-			return fmt.Errorf("failed to base64 decode extra configuration: %w", err)
+			// If it can't be base64 decoded, assume it's already decrypted and skip
+			continue
 		}
 
 		decryptedValue, err := c.secrets.Decrypt(ctx, encryptedValue)

pkg/services/ngalert/remote/alertmanager.go

@@ -50,6 +50,7 @@ func NoopAutogenFn(_ context.Context, _ log.Logger, _ int64, _ *apimodels.Postab
 
 type Crypto interface {
 	Decrypt(ctx context.Context, payload []byte) ([]byte, error)
+	DecryptExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error
 }
 
 type Alertmanager struct {
@@ -282,10 +283,16 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 	if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
 		return err
 	}
+
 	decryptedCfg, err := am.decryptConfiguration(ctx, c)
 	if err != nil {
 		return err
 	}
+
+	// Decrypt and merge extra configs
+	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+		return fmt.Errorf("unable to merge extra configurations: %w", err)
+	}
 	rawDecrypted, err := json.Marshal(decryptedCfg)
 	if err != nil {
 		return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
@@ -297,7 +304,7 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 		return nil
 	}
 
-	return am.sendConfiguration(ctx, decryptedCfg, config.ConfigurationHash, config.CreatedAt, am.isDefaultConfiguration(configHash))
+	return am.sendConfiguration(ctx, decryptedCfg, fmt.Sprintf("%x", configHash), config.CreatedAt, am.isDefaultConfiguration(configHash))
 }
 
 func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) bool {
@@ -342,6 +349,27 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
 	}
 }
 
+// mergeExtraConfigs decrypts and applies merged configuration if extra configs exist.
+func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error {
+	if len(config.ExtraConfigs) == 0 {
+		return nil
+	}
+
+	if err := am.crypto.DecryptExtraConfigs(ctx, config); err != nil {
+		return fmt.Errorf("unable to decrypt extra configs: %w", err)
+	}
+
+	mergeResult, err := config.GetMergedAlertmanagerConfig()
+	if err != nil {
+		return fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
+	}
+	config.AlertmanagerConfig = mergeResult.Config
+	// Clear ExtraConfigs to avoid re-processing them later
+	config.ExtraConfigs = nil
+
+	return nil
+}
+
 func (am *Alertmanager) sendConfiguration(ctx context.Context, decrypted *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
 	am.metrics.ConfigSyncsTotal.Inc()
 	if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(
@@ -380,13 +408,6 @@ func (am *Alertmanager) SendState(ctx context.Context) error {
 
 // SaveAndApplyConfig decrypts and sends a configuration to the remote Alertmanager.
 func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig) error {
-	// Get the hash for the encrypted configuration.
-	rawCfg, err := json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
-
 	// Add auto-generated routes and decrypt before sending.
 	if err := am.autogenFn(ctx, am.log, am.orgID, &cfg.AlertmanagerConfig, false); err != nil {
 		return err
@@ -396,6 +417,16 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 		return err
 	}
 
+	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+		return fmt.Errorf("unable to merge extra configurations: %w", err)
+	}
+
+	rawCfg, err := json.Marshal(decryptedCfg)
+	if err != nil {
+		return err
+	}
+	hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
+
 	return am.sendConfiguration(ctx, decryptedCfg, hash, time.Now().Unix(), false)
 }
 

pkg/services/ngalert/remote/alertmanager_test.go

@@ -11,12 +11,15 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/go-openapi/strfmt"
 	amv2 "github.com/prometheus/alertmanager/api/v2/models"
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
@@ -361,6 +364,15 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, testAutogenFn(nil, nil, 0, &cfgWithAutogenRoutes.AlertmanagerConfig, false))
 
+	// Calculate hashes for expected configurations
+	cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret)
+	require.NoError(t, err)
+	cfgWithDecryptedSecretHash := fmt.Sprintf("%x", md5.Sum(cfgWithDecryptedSecretBytes))
+
+	cfgWithAutogenRoutesBytes, err := json.Marshal(cfgWithAutogenRoutes)
+	require.NoError(t, err)
+	cfgWithAutogenRoutesHash := fmt.Sprintf("%x", md5.Sum(cfgWithAutogenRoutesBytes))
+
 	tests := []struct {
 		name           string
 		config         string
@@ -402,6 +414,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 			NoopAutogenFn,
 			&client.UserGrafanaConfig{
 				GrafanaAlertmanagerConfig: cfgWithDecryptedSecret,
+				Hash:                      cfgWithDecryptedSecretHash,
 			},
 			nil,
 		},
@@ -411,6 +424,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 			testAutogenFn,
 			&client.UserGrafanaConfig{
 				GrafanaAlertmanagerConfig: cfgWithAutogenRoutes,
+				Hash:                      cfgWithAutogenRoutesHash,
 			},
 			nil,
 		},
@@ -561,6 +575,210 @@ func Test_isDefaultConfiguration(t *testing.T) {
 	}
 }
 
+func TestApplyConfigWithExtraConfigs(t *testing.T) {
+	const tenantID = "test"
+
+	var configSent client.UserGrafanaConfig
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader))
+		require.Equal(t, "true", r.Header.Get(client.RemoteAlertmanagerHeader))
+
+		if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/config") {
+			require.NoError(t, json.NewDecoder(r.Body).Decode(&configSent))
+		}
+
+		w.Header().Add("content-type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"}))
+	}))
+	defer server.Close()
+
+	var cfg apimodels.PostableUserConfig
+	require.NoError(t, json.Unmarshal([]byte(testGrafanaConfig), &cfg))
+
+	cfg.ExtraConfigs = []apimodels.ExtraConfiguration{
+		{
+			Identifier: "test-external",
+			MergeMatchers: []*labels.Matcher{
+				{
+					Type:  labels.MatchEqual,
+					Name:  "test",
+					Value: "value",
+				},
+			},
+			TemplateFiles: map[string]string{},
+			AlertmanagerConfig: `global:
+  smtp_smarthost: localhost:587
+  smtp_from: alerts@grafana.com
+route:
+  receiver: extra-receiver
+receivers:
+  - name: extra-receiver
+    email_configs:
+      - to: alerts@grafana.com`,
+		},
+	}
+
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
+	tc := notifier.NewCrypto(secretsService, nil, log.NewNopLogger())
+	ctx := context.Background()
+
+	c := AlertmanagerConfig{
+		OrgID:         1,
+		TenantID:      tenantID,
+		URL:           server.URL,
+		DefaultConfig: defaultGrafanaConfig,
+		PromoteConfig: true,
+	}
+
+	store := ngfakes.NewFakeKVStore(t)
+	fstore := notifier.NewFileStore(1, store)
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.SilencesFilename, ""))
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.NotificationLogFilename, ""))
+
+	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
+	am, err := NewAlertmanager(ctx, c, fstore, tc, NoopAutogenFn, m, tracing.InitializeTracerForTest())
+	require.NoError(t, err)
+
+	err = am.SaveAndApplyConfig(ctx, &cfg)
+	require.NoError(t, err)
+
+	require.Equal(t, len(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers), 2)
+
+	var extraReceiver *apimodels.PostableApiReceiver
+	for _, rcv := range configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers {
+		if rcv.Name == "extra-receiver" {
+			extraReceiver = rcv
+			break
+		}
+	}
+	require.NotNil(t, extraReceiver)
+	require.Len(t, extraReceiver.EmailConfigs, 1)
+	require.Equal(t, "alerts@grafana.com", extraReceiver.EmailConfigs[0].To)
+
+	// Verify the config hash
+	expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
+	require.NoError(t, err)
+	expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
+	require.Equal(t, expectedHash, configSent.Hash)
+}
+
+func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) {
+	const tenantID = "test"
+
+	var configSent client.UserGrafanaConfig
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader))
+		require.Equal(t, "true", r.Header.Get(client.RemoteAlertmanagerHeader))
+
+		if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/config") {
+			require.NoError(t, json.NewDecoder(r.Body).Decode(&configSent))
+		} else if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/config") {
+			// If this is a GET method, Grafana requests the current configuration to compare.
+			// Return an empty config to ensure it gets replaced
+			w.Header().Add("content-type", "application/json")
+			require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{
+				GrafanaAlertmanagerConfig: &apimodels.PostableUserConfig{},
+			}))
+			return
+		}
+
+		w.Header().Add("content-type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"}))
+	}))
+	defer server.Close()
+
+	cfg := apimodels.PostableUserConfig{
+		AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
+			Config: apimodels.Config{
+				Route: &apimodels.Route{
+					Receiver: "grafana-default-email",
+				},
+			},
+			Receivers: []*apimodels.PostableApiReceiver{
+				{
+					Receiver: config.Receiver{Name: "grafana-default-email"},
+					PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{
+						GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{
+							{
+								Name:     "email receiver",
+								Type:     "email",
+								Settings: apimodels.RawMessage(`{"addresses":"<example@email.com>"}`),
+							},
+						},
+					},
+				},
+			},
+		},
+		ExtraConfigs: []apimodels.ExtraConfiguration{
+			{
+				Identifier: "test-external",
+				MergeMatchers: []*labels.Matcher{
+					{
+						Type:  labels.MatchEqual,
+						Name:  "test",
+						Value: "test",
+					},
+				},
+				AlertmanagerConfig: `global:
+  smtp_smarthost: localhost:587
+  smtp_from: alerts@grafana.com
+route:
+  receiver: extra-receiver
+receivers:
+  - name: extra-receiver
+    email_configs:
+      - to: alerts@grafana.com`,
+			},
+		},
+	}
+
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
+	tc := notifier.NewCrypto(secretsService, nil, log.NewNopLogger())
+	ctx := context.Background()
+
+	// Encrypt extra configs since this tests the database path
+	err := tc.EncryptExtraConfigs(ctx, &cfg)
+	require.NoError(t, err)
+
+	c := AlertmanagerConfig{
+		OrgID:         1,
+		TenantID:      tenantID,
+		URL:           server.URL,
+		DefaultConfig: defaultGrafanaConfig,
+		PromoteConfig: true,
+	}
+
+	store := ngfakes.NewFakeKVStore(t)
+	fstore := notifier.NewFileStore(1, store)
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.SilencesFilename, ""))
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.NotificationLogFilename, ""))
+
+	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
+	am, err := NewAlertmanager(ctx, c, fstore, tc, NoopAutogenFn, m, tracing.InitializeTracerForTest())
+	require.NoError(t, err)
+
+	configJSON, err := json.Marshal(cfg)
+	require.NoError(t, err)
+	config := &ngmodels.AlertConfiguration{
+		AlertmanagerConfiguration: string(configJSON),
+	}
+
+	err = am.CompareAndSendConfiguration(ctx, config)
+	require.NoError(t, err)
+
+	require.Equal(t, len(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers), 2)
+	found := slices.ContainsFunc(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers, func(rcv *apimodels.PostableApiReceiver) bool {
+		return strings.Contains(rcv.Name, "extra-receiver")
+	})
+	require.True(t, found)
+
+	// Verify the config hash
+	expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
+	require.NoError(t, err)
+	expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
+	require.Equal(t, expectedHash, configSent.Hash)
+}
+
 func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
@@ -705,7 +923,10 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 		require.NoError(t, err)
 
 		require.JSONEq(t, testGrafanaConfigWithSecret, string(got))
-		require.Equal(t, fmt.Sprintf("%x", md5.Sum(encryptedConfig)), config.Hash)
+
+		// Verify that the hash is calculated from the final configuration, including simplified routing
+		expectedHash := fmt.Sprintf("%x", md5.Sum(got))
+		require.Equal(t, expectedHash, config.Hash, "Hash should be calculated from the final processed configuration")
 		require.False(t, config.Default)
 
 		// An error while adding auto-generated rutes should be returned.

pkg/services/ngalert/remote/client/alertmanager_configuration.go

@@ -3,10 +3,10 @@ package client
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 
+	"github.com/grafana/alerting/definition"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 )
 
@@ -53,7 +53,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana
 }
 
 func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
-	payload, err := json.Marshal(&UserGrafanaConfig{
+	payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{
 		GrafanaAlertmanagerConfig: cfg,
 		Hash:                      hash,
 		CreatedAt:                 createdAt,