Alerting: Send merged configuration to the remote alertmanager (#107004)

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -1051,21 +1051,9 @@ func (c *GettableApiAlertingConfig) UnmarshalYAML(value *yaml.Node) error {
 func (c *GettableApiAlertingConfig) validate() error {
 	receivers := make(map[string]struct{}, len(c.Receivers))
 
-	var hasGrafReceivers, hasAMReceivers bool
-	for _, r := range c.Receivers {
-		receivers[r.Name] = struct{}{}
-		switch r.Type() {
-		case GrafanaReceiverType:
-			hasGrafReceivers = true
-		case AlertmanagerReceiverType:
-			hasAMReceivers = true
-		default:
-			continue
-		}
-	}
-
-	if hasGrafReceivers && hasAMReceivers {
-		return fmt.Errorf("cannot mix Alertmanager & Grafana receiver types")
+	// Populate the receivers map with defined receiver names
+	for _, receiver := range c.Receivers {
+		receivers[receiver.Name] = struct{}{}
 	}
 
 	for _, receiver := range AllReceivers(c.Route.AsAMRoute()) {

pkg/services/ngalert/notifier/crypto.go

@@ -256,9 +256,11 @@ func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *de
 
 func (c *alertmanagerCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
 	for i := range config.ExtraConfigs {
+		// Check if the config is encrypted by trying to base64 decode it
 		encryptedValue, err := base64.StdEncoding.DecodeString(config.ExtraConfigs[i].AlertmanagerConfig)
 		if err != nil {
-			return fmt.Errorf("failed to base64 decode extra configuration: %w", err)
+			// If it can't be base64 decoded, assume it's already decrypted and skip
+			continue
 		}
 
 		decryptedValue, err := c.secrets.Decrypt(ctx, encryptedValue)

pkg/services/ngalert/remote/alertmanager.go

@@ -50,6 +50,7 @@ func NoopAutogenFn(_ context.Context, _ log.Logger, _ int64, _ *apimodels.Postab
 
 type Crypto interface {
 	Decrypt(ctx context.Context, payload []byte) ([]byte, error)
+	DecryptExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error
 }
 
 type Alertmanager struct {
@@ -282,10 +283,16 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 	if err := am.autogenFn(ctx, am.log, am.orgID, &c.AlertmanagerConfig, true); err != nil {
 		return err
 	}
+
 	decryptedCfg, err := am.decryptConfiguration(ctx, c)
 	if err != nil {
 		return err
 	}
+
+	// Decrypt and merge extra configs
+	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+		return fmt.Errorf("unable to merge extra configurations: %w", err)
+	}
 	rawDecrypted, err := json.Marshal(decryptedCfg)
 	if err != nil {
 		return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
@@ -297,7 +304,7 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 		return nil
 	}
 
-	return am.sendConfiguration(ctx, decryptedCfg, config.ConfigurationHash, config.CreatedAt, am.isDefaultConfiguration(configHash))
+	return am.sendConfiguration(ctx, decryptedCfg, fmt.Sprintf("%x", configHash), config.CreatedAt, am.isDefaultConfiguration(configHash))
 }
 
 func (am *Alertmanager) isDefaultConfiguration(configHash [16]byte) bool {
@@ -342,6 +349,27 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
 	}
 }
 
+// mergeExtraConfigs decrypts and applies merged configuration if extra configs exist.
+func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error {
+	if len(config.ExtraConfigs) == 0 {
+		return nil
+	}
+
+	if err := am.crypto.DecryptExtraConfigs(ctx, config); err != nil {
+		return fmt.Errorf("unable to decrypt extra configs: %w", err)
+	}
+
+	mergeResult, err := config.GetMergedAlertmanagerConfig()
+	if err != nil {
+		return fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
+	}
+	config.AlertmanagerConfig = mergeResult.Config
+	// Clear ExtraConfigs to avoid re-processing them later
+	config.ExtraConfigs = nil
+
+	return nil
+}
+
 func (am *Alertmanager) sendConfiguration(ctx context.Context, decrypted *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
 	am.metrics.ConfigSyncsTotal.Inc()
 	if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(
@@ -380,13 +408,6 @@ func (am *Alertmanager) SendState(ctx context.Context) error {
 
 // SaveAndApplyConfig decrypts and sends a configuration to the remote Alertmanager.
 func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig) error {
-	// Get the hash for the encrypted configuration.
-	rawCfg, err := json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
-
 	// Add auto-generated routes and decrypt before sending.
 	if err := am.autogenFn(ctx, am.log, am.orgID, &cfg.AlertmanagerConfig, false); err != nil {
 		return err
@@ -396,6 +417,16 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 		return err
 	}
 
+	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+		return fmt.Errorf("unable to merge extra configurations: %w", err)
+	}
+
+	rawCfg, err := json.Marshal(decryptedCfg)
+	if err != nil {
+		return err
+	}
+	hash := fmt.Sprintf("%x", md5.Sum(rawCfg))
+
 	return am.sendConfiguration(ctx, decryptedCfg, hash, time.Now().Unix(), false)
 }
 

pkg/services/ngalert/remote/alertmanager_test.go

@@ -11,12 +11,15 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/go-openapi/strfmt"
 	amv2 "github.com/prometheus/alertmanager/api/v2/models"
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
@@ -361,6 +364,15 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, testAutogenFn(nil, nil, 0, &cfgWithAutogenRoutes.AlertmanagerConfig, false))
 
+	// Calculate hashes for expected configurations
+	cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret)
+	require.NoError(t, err)
+	cfgWithDecryptedSecretHash := fmt.Sprintf("%x", md5.Sum(cfgWithDecryptedSecretBytes))
+
+	cfgWithAutogenRoutesBytes, err := json.Marshal(cfgWithAutogenRoutes)
+	require.NoError(t, err)
+	cfgWithAutogenRoutesHash := fmt.Sprintf("%x", md5.Sum(cfgWithAutogenRoutesBytes))
+
 	tests := []struct {
 		name           string
 		config         string
@@ -402,6 +414,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 			NoopAutogenFn,
 			&client.UserGrafanaConfig{
 				GrafanaAlertmanagerConfig: cfgWithDecryptedSecret,
+				Hash:                      cfgWithDecryptedSecretHash,
 			},
 			nil,
 		},
@@ -411,6 +424,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 			testAutogenFn,
 			&client.UserGrafanaConfig{
 				GrafanaAlertmanagerConfig: cfgWithAutogenRoutes,
+				Hash:                      cfgWithAutogenRoutesHash,
 			},
 			nil,
 		},
@@ -561,6 +575,210 @@ func Test_isDefaultConfiguration(t *testing.T) {
 	}
 }
 
+func TestApplyConfigWithExtraConfigs(t *testing.T) {
+	const tenantID = "test"
+
+	var configSent client.UserGrafanaConfig
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader))
+		require.Equal(t, "true", r.Header.Get(client.RemoteAlertmanagerHeader))
+
+		if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/config") {
+			require.NoError(t, json.NewDecoder(r.Body).Decode(&configSent))
+		}
+
+		w.Header().Add("content-type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"}))
+	}))
+	defer server.Close()
+
+	var cfg apimodels.PostableUserConfig
+	require.NoError(t, json.Unmarshal([]byte(testGrafanaConfig), &cfg))
+
+	cfg.ExtraConfigs = []apimodels.ExtraConfiguration{
+		{
+			Identifier: "test-external",
+			MergeMatchers: []*labels.Matcher{
+				{
+					Type:  labels.MatchEqual,
+					Name:  "test",
+					Value: "value",
+				},
+			},
+			TemplateFiles: map[string]string{},
+			AlertmanagerConfig: `global:
+  smtp_smarthost: localhost:587
+  smtp_from: alerts@grafana.com
+route:
+  receiver: extra-receiver
+receivers:
+  - name: extra-receiver
+    email_configs:
+      - to: alerts@grafana.com`,
+		},
+	}
+
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
+	tc := notifier.NewCrypto(secretsService, nil, log.NewNopLogger())
+	ctx := context.Background()
+
+	c := AlertmanagerConfig{
+		OrgID:         1,
+		TenantID:      tenantID,
+		URL:           server.URL,
+		DefaultConfig: defaultGrafanaConfig,
+		PromoteConfig: true,
+	}
+
+	store := ngfakes.NewFakeKVStore(t)
+	fstore := notifier.NewFileStore(1, store)
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.SilencesFilename, ""))
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.NotificationLogFilename, ""))
+
+	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
+	am, err := NewAlertmanager(ctx, c, fstore, tc, NoopAutogenFn, m, tracing.InitializeTracerForTest())
+	require.NoError(t, err)
+
+	err = am.SaveAndApplyConfig(ctx, &cfg)
+	require.NoError(t, err)
+
+	require.Equal(t, len(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers), 2)
+
+	var extraReceiver *apimodels.PostableApiReceiver
+	for _, rcv := range configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers {
+		if rcv.Name == "extra-receiver" {
+			extraReceiver = rcv
+			break
+		}
+	}
+	require.NotNil(t, extraReceiver)
+	require.Len(t, extraReceiver.EmailConfigs, 1)
+	require.Equal(t, "alerts@grafana.com", extraReceiver.EmailConfigs[0].To)
+
+	// Verify the config hash
+	expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
+	require.NoError(t, err)
+	expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
+	require.Equal(t, expectedHash, configSent.Hash)
+}
+
+func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) {
+	const tenantID = "test"
+
+	var configSent client.UserGrafanaConfig
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, tenantID, r.Header.Get(client.MimirTenantHeader))
+		require.Equal(t, "true", r.Header.Get(client.RemoteAlertmanagerHeader))
+
+		if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/config") {
+			require.NoError(t, json.NewDecoder(r.Body).Decode(&configSent))
+		} else if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/config") {
+			// If this is a GET method, Grafana requests the current configuration to compare.
+			// Return an empty config to ensure it gets replaced
+			w.Header().Add("content-type", "application/json")
+			require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{
+				GrafanaAlertmanagerConfig: &apimodels.PostableUserConfig{},
+			}))
+			return
+		}
+
+		w.Header().Add("content-type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"}))
+	}))
+	defer server.Close()
+
+	cfg := apimodels.PostableUserConfig{
+		AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
+			Config: apimodels.Config{
+				Route: &apimodels.Route{
+					Receiver: "grafana-default-email",
+				},
+			},
+			Receivers: []*apimodels.PostableApiReceiver{
+				{
+					Receiver: config.Receiver{Name: "grafana-default-email"},
+					PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{
+						GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{
+							{
+								Name:     "email receiver",
+								Type:     "email",
+								Settings: apimodels.RawMessage(`{"addresses":"<example@email.com>"}`),
+							},
+						},
+					},
+				},
+			},
+		},
+		ExtraConfigs: []apimodels.ExtraConfiguration{
+			{
+				Identifier: "test-external",
+				MergeMatchers: []*labels.Matcher{
+					{
+						Type:  labels.MatchEqual,
+						Name:  "test",
+						Value: "test",
+					},
+				},
+				AlertmanagerConfig: `global:
+  smtp_smarthost: localhost:587
+  smtp_from: alerts@grafana.com
+route:
+  receiver: extra-receiver
+receivers:
+  - name: extra-receiver
+    email_configs:
+      - to: alerts@grafana.com`,
+			},
+		},
+	}
+
+	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
+	tc := notifier.NewCrypto(secretsService, nil, log.NewNopLogger())
+	ctx := context.Background()
+
+	// Encrypt extra configs since this tests the database path
+	err := tc.EncryptExtraConfigs(ctx, &cfg)
+	require.NoError(t, err)
+
+	c := AlertmanagerConfig{
+		OrgID:         1,
+		TenantID:      tenantID,
+		URL:           server.URL,
+		DefaultConfig: defaultGrafanaConfig,
+		PromoteConfig: true,
+	}
+
+	store := ngfakes.NewFakeKVStore(t)
+	fstore := notifier.NewFileStore(1, store)
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.SilencesFilename, ""))
+	require.NoError(t, store.Set(ctx, c.OrgID, "alertmanager", notifier.NotificationLogFilename, ""))
+
+	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
+	am, err := NewAlertmanager(ctx, c, fstore, tc, NoopAutogenFn, m, tracing.InitializeTracerForTest())
+	require.NoError(t, err)
+
+	configJSON, err := json.Marshal(cfg)
+	require.NoError(t, err)
+	config := &ngmodels.AlertConfiguration{
+		AlertmanagerConfiguration: string(configJSON),
+	}
+
+	err = am.CompareAndSendConfiguration(ctx, config)
+	require.NoError(t, err)
+
+	require.Equal(t, len(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers), 2)
+	found := slices.ContainsFunc(configSent.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers, func(rcv *apimodels.PostableApiReceiver) bool {
+		return strings.Contains(rcv.Name, "extra-receiver")
+	})
+	require.True(t, found)
+
+	// Verify the config hash
+	expectedConfigBytes, err := json.Marshal(configSent.GrafanaAlertmanagerConfig)
+	require.NoError(t, err)
+	expectedHash := fmt.Sprintf("%x", md5.Sum(expectedConfigBytes))
+	require.Equal(t, expectedHash, configSent.Hash)
+}
+
 func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
@@ -705,7 +923,10 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 		require.NoError(t, err)
 
 		require.JSONEq(t, testGrafanaConfigWithSecret, string(got))
-		require.Equal(t, fmt.Sprintf("%x", md5.Sum(encryptedConfig)), config.Hash)
+
+		// Verify that the hash is calculated from the final configuration, including simplified routing
+		expectedHash := fmt.Sprintf("%x", md5.Sum(got))
+		require.Equal(t, expectedHash, config.Hash, "Hash should be calculated from the final processed configuration")
 		require.False(t, config.Default)
 
 		// An error while adding auto-generated rutes should be returned.

pkg/services/ngalert/remote/client/alertmanager_configuration.go

@@ -3,10 +3,10 @@ package client
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 
+	"github.com/grafana/alerting/definition"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 )
 
@@ -53,7 +53,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana
 }
 
 func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error {
-	payload, err := json.Marshal(&UserGrafanaConfig{
+	payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{
 		GrafanaAlertmanagerConfig: cfg,
 		Hash:                      hash,
 		CreatedAt:                 createdAt,

pkg/tests/alertmanager/alertmanager_scenario.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/grafana/e2e"
 	gapi "github.com/grafana/grafana-api-golang-client"
+	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/stretchr/testify/require"
 )
 
@@ -43,6 +44,7 @@ type AlertmanagerScenario struct {
 	Webhook  *WebhookService
 	Postgres *PostgresService
 	Loki     *LokiService
+	Mimir    *MimirService
 }
 
 func NewAlertmanagerScenario() (*AlertmanagerScenario, error) {
@@ -381,3 +383,10 @@ func mapInstancePeers(is []string) map[string][]string {
 
 	return mIs
 }
+
+func (s *AlertmanagerScenario) NewMimirClient(tenantID string) (client.MimirClient, error) {
+	if s.Mimir == nil {
+		return nil, fmt.Errorf("mimir service not started")
+	}
+	return NewMimirClient("http://"+s.Mimir.HTTPEndpoint(), tenantID)
+}

pkg/tests/alertmanager/mimir.go

@@ -0,0 +1,67 @@
+package alertmanager
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/grafana/e2e"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	mimirImage = "grafana/mimir:r348-017076d8"
+
+	mimirBinary   = "/bin/mimir"
+	mimirHTTPPort = 33667
+	mimirGRPCPort = 33668
+)
+
+type MimirService struct {
+	*e2e.HTTPService
+}
+
+func NewMimirService(name string) *MimirService {
+	flags := map[string]string{
+		"-target":                                                  "alertmanager",
+		"-server.http-listen-port":                                 fmt.Sprintf("%d", mimirHTTPPort),
+		"-server.grpc-listen-port":                                 fmt.Sprintf("%d", mimirGRPCPort),
+		"-alertmanager.web.external-url":                           "http://localhost:8080/alertmanager",
+		"-alertmanager-storage.backend":                            "filesystem",
+		"-alertmanager-storage.filesystem.dir":                     "/tmp/mimir/alertmanager",
+		"-alertmanager.grafana-alertmanager-compatibility-enabled": "true",
+	}
+
+	return &MimirService{
+		HTTPService: e2e.NewHTTPService(
+			name,
+			mimirImage,
+			e2e.NewCommandWithoutEntrypoint(mimirBinary, e2e.BuildArgs(flags)...),
+			e2e.NewHTTPReadinessProbe(mimirHTTPPort, "/ready", 200, 299),
+			mimirHTTPPort,
+		),
+	}
+}
+
+func NewMimirClient(mimirURL, tenantID string) (client.MimirClient, error) {
+	u, err := url.Parse(mimirURL)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &client.Config{
+		URL:      u,
+		TenantID: tenantID,
+		Password: "", // No password needed for test
+		Logger:   log.NewNopLogger(),
+	}
+
+	registry := prometheus.NewRegistry()
+	metrics := metrics.NewRemoteAlertmanagerMetrics(registry)
+	tracer := tracing.InitializeTracerForTest()
+
+	return client.New(cfg, metrics, tracer)
+}

pkg/tests/api/alerting/api_remote_alertmanager_test.go

@@ -0,0 +1,262 @@
+package alerting
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/tests/alertmanager"
+	"github.com/grafana/grafana/pkg/tests/testinfra"
+)
+
+// TestIntegrationRemoteAlertmanagerConfigUpload tests that when we post an alertmanager
+// configuration to Grafana with remote alertmanager enabled, it gets uploaded to the remote Mimir.
+func TestIntegrationRemoteAlertmanagerConfigUpload(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	s, err := alertmanager.NewAlertmanagerScenario()
+	require.NoError(t, err)
+	defer s.Close()
+
+	s.Mimir = alertmanager.NewMimirService("mimir")
+	require.NoError(t, s.StartAndWaitReady(s.Mimir))
+
+	mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
+
+	dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
+		AppModeProduction:     true,
+		EnableFeatureToggles: []string{
+			"alertmanagerRemotePrimary",
+			"alertingImportAlertmanagerAPI",
+		},
+		RemoteAlertmanagerURL: mimirEndpoint,
+	})
+
+	grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
+	mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
+	require.NoError(t, err)
+
+	// Wait for Grafana to be ready
+	require.Eventually(t, func() bool {
+		_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+		return status == http.StatusOK
+	}, 30*time.Second, time.Second, "Grafana failed to start")
+
+	// Check that the initial Mimir config contains the default Grafana configuration
+	initialMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, initialMimirConfig) // Grafana automatically syncs default config to remote alertmanager
+	require.NotNil(t, initialMimirConfig.GrafanaAlertmanagerConfig)
+
+	// Initially there is just the default grafana-default-email receiver
+	receivers := initialMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 1)
+	require.Equal(t, "grafana-default-email", receivers[0].Name)
+
+	// Now upload a new extra config and check that it gets uploaded to Mimir
+	testAlertmanagerConfigYAML := `
+route:
+  group_by: ['alertname'] 
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 1h
+  receiver: extra-slack
+
+receivers:
+- name: extra-slack
+  slack_configs:
+  - api_url: 'http://localhost/slack'
+    channel: '#alerts'
+    title: 'Alerts'
+`
+
+	headers := map[string]string{
+		"Content-Type":                         "application/yaml",
+		"X-Grafana-Alerting-Config-Identifier": "external-system",
+		"X-Grafana-Alerting-Merge-Matchers":    "environment=production,team=backend",
+	}
+
+	amConfig := apimodels.AlertmanagerUserConfig{
+		AlertmanagerConfig: testAlertmanagerConfigYAML,
+		TemplateFiles: map[string]string{
+			"test.tmpl": `{{ define "test.template" }}Test template for remote sync{{ end }}`,
+		},
+	}
+
+	// Post the configuration to Grafana
+	response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
+	require.Equal(t, "success", response.Status)
+
+	_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+	require.Equal(t, http.StatusOK, status)
+
+	// Check that the configuration was successfully sent to Mimir and contains the new receiver
+	finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, finalMimirConfig)
+	require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
+
+	receivers = finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 2)
+
+	var foundDefault, foundExtraSlack bool
+	for _, receiver := range receivers {
+		switch receiver.Name {
+		case "grafana-default-email":
+			foundDefault = true
+			require.Len(t, receiver.GrafanaManagedReceivers, 1)
+			require.Equal(t, "email receiver", receiver.GrafanaManagedReceivers[0].Name)
+			require.Equal(t, "email", receiver.GrafanaManagedReceivers[0].Type)
+		case "extra-slack":
+			foundExtraSlack = true
+			require.Len(t, receiver.SlackConfigs, 1)
+			require.NotNil(t, receiver.SlackConfigs[0].APIURL)
+			require.Equal(t, "#alerts", receiver.SlackConfigs[0].Channel)
+		}
+	}
+	require.True(t, foundDefault, "Default receiver not found")
+	require.True(t, foundExtraSlack, "Extra slack receiver not found")
+}
+
+// TestIntegrationRemoteAlertmanagerHistoricalConfigActivation tests that when we activate
+// a historical alertmanager configuration with extra configs, it gets properly decrypted
+// and uploaded to the remote Mimir.
+func TestIntegrationRemoteAlertmanagerHistoricalConfigActivation(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	s, err := alertmanager.NewAlertmanagerScenario()
+	require.NoError(t, err)
+	defer s.Close()
+
+	s.Mimir = alertmanager.NewMimirService("mimir")
+	require.NoError(t, s.StartAndWaitReady(s.Mimir))
+
+	mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
+
+	dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
+		AppModeProduction:     true,
+		EnableFeatureToggles: []string{
+			"alertmanagerRemotePrimary",
+			"alertingImportAlertmanagerAPI",
+		},
+		RemoteAlertmanagerURL: mimirEndpoint,
+	})
+
+	grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
+	mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+		return status == http.StatusOK
+	}, 30*time.Second, time.Second, "Grafana failed to start")
+
+	// Upload configuration with extra configs
+	testAlertmanagerConfigYAML := `
+route:
+  group_by: ['alertname'] 
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 1h
+  receiver: old-slack
+
+receivers:
+- name: old-slack
+  slack_configs:
+  - api_url: 'http://localhost/slack'
+    channel: '#alerts'
+`
+
+	headers := map[string]string{
+		"Content-Type":                         "application/yaml",
+		"X-Grafana-Alerting-Config-Identifier": "historical-system",
+		"X-Grafana-Alerting-Merge-Matchers":    "environment=test,team=platform",
+	}
+
+	amConfig := apimodels.AlertmanagerUserConfig{
+		AlertmanagerConfig: testAlertmanagerConfigYAML,
+		TemplateFiles: map[string]string{
+			"historical.tmpl": `{{ define "historical.template" }}Historical template{{ end }}`,
+		},
+	}
+
+	response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
+	require.Equal(t, "success", response.Status)
+
+	// Get the configuration history to find the most recent config
+	historyResponse := getAlertmanagerConfigHistory(t, apiClient)
+	require.NotEmpty(t, historyResponse)
+
+	var mostRecentID int64
+	for _, entry := range historyResponse {
+		if entry.ID > mostRecentID {
+			mostRecentID = entry.ID
+		}
+	}
+	require.Greater(t, mostRecentID, int64(0), "Should have found a historical configuration")
+
+	// Activate the historical configuration
+	activateHistoricalConfiguration(t, apiClient, mostRecentID)
+
+	// Verify the configuration
+	finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, finalMimirConfig)
+	require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
+
+	receivers := finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 2)
+
+	found := false
+	for _, receiver := range receivers {
+		if receiver.Name == "old-slack" {
+			found = true
+			require.Len(t, receiver.SlackConfigs, 1)
+			break
+		}
+	}
+	require.True(t, found)
+}
+
+func getAlertmanagerConfigHistory(t *testing.T, client apiClient) []apimodels.GettableHistoricUserConfig {
+	t.Helper()
+	u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history", client.url))
+	require.NoError(t, err)
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	require.NoError(t, err)
+
+	history, _, _ := sendRequestJSON[[]apimodels.GettableHistoricUserConfig](t, req, http.StatusOK)
+	return history
+}
+
+func activateHistoricalConfiguration(t *testing.T, client apiClient, configID int64) {
+	t.Helper()
+	u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history/%d/_activate", client.url, configID))
+	require.NoError(t, err)
+
+	req, err := http.NewRequest(http.MethodPost, u.String(), nil)
+	require.NoError(t, err)
+
+	response, statusCode, body := sendRequestJSON[map[string]string](t, req, http.StatusAccepted)
+	if statusCode != http.StatusAccepted {
+		t.Fatalf("Expected status code %d but got %d. Response body: %s", http.StatusAccepted, statusCode, body)
+	}
+	require.Equal(t, "configuration activated", response["message"])
+}

pkg/tests/testinfra/testinfra.go

@@ -477,6 +477,17 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) {
 		_, err = grafanaComSection.NewKey("api_url", opts.GrafanaComAPIURL)
 		require.NoError(t, err)
 	}
+
+	if opts.RemoteAlertmanagerURL != "" {
+		remoteAlertmanagerSection, err := getOrCreateSection("remote.alertmanager")
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("enabled", "true")
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("url", opts.RemoteAlertmanagerURL)
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("tenant", "1")
+		require.NoError(t, err)
+	}
 	if opts.GrafanaComSSOAPIToken != "" {
 		grafanaComSection, err := getOrCreateSection("grafana_com")
 		require.NoError(t, err)
@@ -571,6 +582,9 @@ type GrafanaOpts struct {
 
 	// When "unified-grpc" is selected it will also start the grpc server
 	APIServerStorageType options.StorageType
+
+	// Remote alertmanager configuration
+	RemoteAlertmanagerURL string
 }
 
 func CreateUser(t *testing.T, store db.DB, cfg *setting.Cfg, cmd user.CreateUserCommand) *user.User {