Alerting: Support saving extra Mimir configurations (#106721)

pkg/services/ngalert/api/tooling/api.json

@@ -1123,6 +1123,26 @@
    },
    "type": "object"
   },
+  "ExtraConfiguration": {
+   "properties": {
+    "alertmanager_config": {
+     "type": "string"
+    },
+    "identifier": {
+     "type": "string"
+    },
+    "merge_matchers": {
+     "$ref": "#/definitions/Matchers"
+    },
+    "template_files": {
+     "additionalProperties": {
+      "type": "string"
+     },
+     "type": "object"
+    }
+   },
+   "type": "object"
+  },
   "Failure": {
    "$ref": "#/definitions/ResponseDetails"
   },
@@ -1854,6 +1874,12 @@
     "alertmanager_config": {
      "$ref": "#/definitions/GettableApiAlertingConfig"
     },
+    "extra_config": {
+     "items": {
+      "$ref": "#/definitions/ExtraConfiguration"
+     },
+     "type": "array"
+    },
     "template_file_provenances": {
      "additionalProperties": {
       "$ref": "#/definitions/Provenance"
@@ -3057,6 +3083,12 @@
     "alertmanager_config": {
      "$ref": "#/definitions/PostableApiAlertingConfig"
     },
+    "extra_config": {
+     "items": {
+      "$ref": "#/definitions/ExtraConfiguration"
+     },
+     "type": "array"
+    },
     "template_files": {
      "additionalProperties": {
       "type": "string"
@@ -3687,7 +3719,6 @@
    "type": "object"
   },
   "Route": {
-   "description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
    "properties": {
     "active_time_intervals": {
      "items": {
@@ -3729,12 +3760,6 @@
      },
      "type": "array"
     },
-    "object_matchers": {
-     "$ref": "#/definitions/ObjectMatchers"
-    },
-    "provenance": {
-     "$ref": "#/definitions/Provenance"
-    },
     "receiver": {
      "type": "string"
     },
@@ -3748,6 +3773,7 @@
      "type": "array"
     }
    },
+   "title": "A Route is a node that contains definitions of how to handle alerts.",
    "type": "object"
   },
   "RouteExport": {

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -13,7 +13,6 @@ import (
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/common/model"
-
 	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/alerting/definition"
@@ -649,31 +648,69 @@ type DatasourceUIDReference struct {
 }
 
 type ExtraConfiguration struct {
-	Identifier         string                    `yaml:"identifier" json:"identifier"`
+	Identifier         string            `yaml:"identifier" json:"identifier"`
-	MergeMatchers      config.Matchers           `yaml:"merge_matchers" json:"merge_matchers"`
+	MergeMatchers      config.Matchers   `yaml:"merge_matchers" json:"merge_matchers"`
-	TemplateFiles      map[string]string         `yaml:"template_files" json:"template_files"`
+	TemplateFiles      map[string]string `yaml:"template_files" json:"template_files"`
-	AlertmanagerConfig PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
+	AlertmanagerConfig string            `yaml:"alertmanager_config" json:"alertmanager_config"`
+}
+
+func (c *ExtraConfiguration) GetAlertmanagerConfig() (PostableApiAlertingConfig, error) {
+	if c.AlertmanagerConfig == "" {
+		return PostableApiAlertingConfig{}, fmt.Errorf("no alertmanager configuration available")
+	}
+
+	var prometheusConfig config.Config
+	if err := yaml.Unmarshal([]byte(c.AlertmanagerConfig), &prometheusConfig); err != nil {
+		return PostableApiAlertingConfig{}, fmt.Errorf("failed to parse alertmanager config: %w", err)
+	}
+
+	return fromPrometheusConfig(prometheusConfig), nil
 }
 
 func (c ExtraConfiguration) Validate() error {
 	if c.Identifier == "" {
 		return errors.New("identifier is required")
 	}
+
 	if len(c.MergeMatchers) == 0 {
 		return errors.New("at least one matcher is required")
 	}
+
 	for _, m := range c.MergeMatchers {
 		if m.Type != labels.MatchEqual {
 			return errors.New("only matchers with type equal are supported")
 		}
 	}
-	err := c.AlertmanagerConfig.Validate()
+
+	// Alertmanager configuration is validated during YAML unmarshalling.
+	am := config.Config{}
+	err := yaml.Unmarshal([]byte(c.AlertmanagerConfig), &am)
 	if err != nil {
 		return fmt.Errorf("invalid alertmanager configuration: %w", err)
 	}
+
 	return nil
 }
 
+func fromPrometheusConfig(prometheusConfig config.Config) PostableApiAlertingConfig {
+	config := PostableApiAlertingConfig{
+		Config: Config{
+			Global:       prometheusConfig.Global,
+			Route:        AsGrafanaRoute(prometheusConfig.Route),
+			InhibitRules: prometheusConfig.InhibitRules,
+			Templates:    prometheusConfig.Templates,
+		},
+	}
+
+	for _, receiver := range prometheusConfig.Receivers {
+		config.Receivers = append(config.Receivers, &PostableApiReceiver{
+			Receiver: receiver,
+		})
+	}
+
+	return config
+}
+
 // swagger:model
 type PostableUserConfig struct {
 	TemplateFiles      map[string]string         `yaml:"template_files" json:"template_files"`
@@ -697,7 +734,13 @@ func (c *PostableUserConfig) GetMergedAlertmanagerConfig() (MergeResult, error)
 	if err := opts.Validate(); err != nil {
 		return MergeResult{}, fmt.Errorf("invalid merge options: %w", err)
 	}
-	return definition.Merge(c.AlertmanagerConfig, mimirCfg.AlertmanagerConfig, opts) // for now support only the first extra config
+
+	mcfg, err := mimirCfg.GetAlertmanagerConfig()
+	if err != nil {
+		return MergeResult{}, fmt.Errorf("failed to get mimir alertmanager config: %w", err)
+	}
+
+	return definition.Merge(c.AlertmanagerConfig, mcfg, opts)
 }
 
 // GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of TemplateDefinitions.
@@ -738,11 +781,6 @@ func (c *PostableUserConfig) UnmarshalJSON(b []byte) error {
 	if len(c.ExtraConfigs) > 1 {
 		return errors.New("only one extra config is supported")
 	}
-	for _, extraConfig := range c.ExtraConfigs {
-		if err := extraConfig.Validate(); err != nil {
-			return fmt.Errorf("extra configuration is invalid: %w", err)
-		}
-	}
 
 	type intermediate struct {
 		AlertmanagerConfig map[string]interface{} `yaml:"alertmanager_config" json:"alertmanager_config"`
@@ -833,6 +871,7 @@ type GettableUserConfig struct {
 	TemplateFiles           map[string]string         `yaml:"template_files" json:"template_files"`
 	TemplateFileProvenances map[string]Provenance     `yaml:"template_file_provenances,omitempty" json:"template_file_provenances,omitempty"`
 	AlertmanagerConfig      GettableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
+	ExtraConfigs            []ExtraConfiguration      `yaml:"extra_config,omitempty" json:"extra_config,omitempty"`
 
 	// amSimple stores a map[string]interface of the decoded alertmanager config.
 	// This enables circumventing the underlying alertmanager secret type

pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go

@@ -259,20 +259,10 @@ func TestPostableUserConfig_GetMergedAlertmanagerConfig(t *testing.T) {
 								Value: "prod",
 							},
 						},
-						AlertmanagerConfig: PostableApiAlertingConfig{
+						AlertmanagerConfig: `route:
-							Config: Config{
+  receiver: mimir-receiver
-								Route: &Route{
+receivers:
-									Receiver: "mimir-receiver",
+  - name: mimir-receiver`,
-								},
-							},
-							Receivers: []*PostableApiReceiver{
-								{
-									Receiver: config.Receiver{
-										Name: "mimir-receiver",
-									},
-								},
-							},
-						},
 					},
 				},
 			},
@@ -285,13 +275,11 @@ func TestPostableUserConfig_GetMergedAlertmanagerConfig(t *testing.T) {
 					{
 						Identifier:    "",
 						MergeMatchers: config.Matchers{},
-						AlertmanagerConfig: PostableApiAlertingConfig{
+						AlertmanagerConfig: `{
-							Config: Config{
+							"route": {
-								Route: &Route{
+								"receiver": "test"
-									Receiver: "test",
+							}
-								},
+						}`,
-							},
-						},
 					},
 				},
 			},

pkg/services/ngalert/api/tooling/post.json

@@ -1123,6 +1123,26 @@
    },
    "type": "object"
   },
+  "ExtraConfiguration": {
+   "properties": {
+    "alertmanager_config": {
+     "type": "string"
+    },
+    "identifier": {
+     "type": "string"
+    },
+    "merge_matchers": {
+     "$ref": "#/definitions/Matchers"
+    },
+    "template_files": {
+     "additionalProperties": {
+      "type": "string"
+     },
+     "type": "object"
+    }
+   },
+   "type": "object"
+  },
   "Failure": {
    "$ref": "#/definitions/ResponseDetails"
   },
@@ -1854,6 +1874,12 @@
     "alertmanager_config": {
      "$ref": "#/definitions/GettableApiAlertingConfig"
     },
+    "extra_config": {
+     "items": {
+      "$ref": "#/definitions/ExtraConfiguration"
+     },
+     "type": "array"
+    },
     "template_file_provenances": {
      "additionalProperties": {
       "$ref": "#/definitions/Provenance"
@@ -3057,6 +3083,12 @@
     "alertmanager_config": {
      "$ref": "#/definitions/PostableApiAlertingConfig"
     },
+    "extra_config": {
+     "items": {
+      "$ref": "#/definitions/ExtraConfiguration"
+     },
+     "type": "array"
+    },
     "template_files": {
      "additionalProperties": {
       "type": "string"

pkg/services/ngalert/api/tooling/spec.json

@@ -5412,6 +5412,26 @@
         }
       }
     },
+    "ExtraConfiguration": {
+      "type": "object",
+      "properties": {
+        "alertmanager_config": {
+          "type": "string"
+        },
+        "identifier": {
+          "type": "string"
+        },
+        "merge_matchers": {
+          "$ref": "#/definitions/Matchers"
+        },
+        "template_files": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "Failure": {
       "$ref": "#/definitions/ResponseDetails"
     },
@@ -6144,6 +6164,12 @@
         "alertmanager_config": {
           "$ref": "#/definitions/GettableApiAlertingConfig"
         },
+        "extra_config": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ExtraConfiguration"
+          }
+        },
         "template_file_provenances": {
           "type": "object",
           "additionalProperties": {
@@ -7348,6 +7374,12 @@
         "alertmanager_config": {
           "$ref": "#/definitions/PostableApiAlertingConfig"
         },
+        "extra_config": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ExtraConfiguration"
+          }
+        },
         "template_files": {
           "type": "object",
           "additionalProperties": {

pkg/services/ngalert/notifier/alertmanager.go

@@ -55,6 +55,7 @@ type alertmanager struct {
 	stateStore           stateStore
 	DefaultConfiguration string
 	decryptFn            alertingNotify.GetDecryptedValueFn
+	crypto               Crypto
 }
 
 // maintenanceOptions represent the options for components that need maintenance on a frequency within the Alertmanager.
@@ -86,7 +87,7 @@ func (m maintenanceOptions) MaintenanceFunc(state alertingNotify.State) (int64,
 
 func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store AlertingStore, stateStore stateStore,
 	peer alertingNotify.ClusterPeer, decryptFn alertingNotify.GetDecryptedValueFn, ns notifications.Service,
-	m *metrics.Alertmanager, featureToggles featuremgmt.FeatureToggles,
+	m *metrics.Alertmanager, featureToggles featuremgmt.FeatureToggles, crypto Crypto,
 ) (*alertmanager, error) {
 	nflog, err := stateStore.GetNotificationLog(ctx)
 	if err != nil {
@@ -152,6 +153,7 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
 		stateStore:           stateStore,
 		logger:               l.New("component", "alertmanager", opts.TenantKey, opts.TenantID), // similar to what the base does
 		decryptFn:            decryptFn,
+		crypto:               crypto,
 	}
 
 	return am, nil
@@ -206,7 +208,13 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 	// Remove autogenerated config from the user config before saving it, may not be necessary as we already remove
 	// the autogenerated config before provenance guard. However, this is low impact and a good safety net.
 	RemoveAutogenConfigIfExists(cfg.AlertmanagerConfig.Route)
-	rawConfig, err := json.Marshal(&cfg)
+
+	err := am.crypto.EncryptExtraConfigs(ctx, cfg)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt external configurations: %w", err)
+	}
+
+	cfgToSave, err := json.Marshal(&cfg)
 	if err != nil {
 		return fmt.Errorf("failed to serialize to the Alertmanager configuration: %w", err)
 	}
@@ -214,7 +222,7 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 	var outerErr error
 	am.Base.WithLock(func() {
 		cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{
-			AlertmanagerConfiguration: string(rawConfig),
+			AlertmanagerConfiguration: string(cfgToSave),
 			ConfigurationVersion:      fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
 			OrgID:                     am.Base.TenantID(),
 			LastApplied:               time.Now().UTC().Unix(),
@@ -342,6 +350,11 @@ func logMergeResult(l log.Logger, m apimodels.MergeResult) {
 // It returns a boolean indicating whether the user config was changed and an error.
 // It is not safe to call concurrently.
 func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, skipInvalid bool) (bool, error) {
+	err := am.crypto.DecryptExtraConfigs(ctx, cfg)
+	if err != nil {
+		return false, fmt.Errorf("failed to decrypt external configurations: %w", err)
+	}
+
 	mergeResult, err := cfg.GetMergedAlertmanagerConfig()
 	if err != nil {
 		return false, fmt.Errorf("failed to get full alertmanager configuration: %w", err)

pkg/services/ngalert/notifier/alertmanager_config.go

@@ -30,6 +30,12 @@ var (
 		errutil.WithPublic(
 			"time interval [Name: {{ .Public.Interval }}] is used by rule",
 		))
+
+	msgAlertmanagerMultipleExtraConfigsUnsupported = "multiple extra configurations are not supported, found another configuration with identifier: {{ .Public.Identifier }}"
+	ErrAlertmanagerMultipleExtraConfigsUnsupported = errutil.Conflict("alerting.notifications.alertmanager.multipleExtraConfigsUnsupported").MustTemplate(
+		msgAlertmanagerMultipleExtraConfigsUnsupported,
+		errutil.WithPublic(msgAlertmanagerMultipleExtraConfigsUnsupported),
+	)
 )
 
 type UnknownReceiverError struct {
@@ -217,11 +223,17 @@ func (moa *MultiOrgAlertmanager) gettableUserConfigFromAMConfigString(ctx contex
 		return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err)
 	}
 
+	err = moa.Crypto.DecryptExtraConfigs(ctx, cfg)
+	if err != nil {
+		return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt external configurations: %w", err)
+	}
+
 	result := definitions.GettableUserConfig{
 		TemplateFiles: cfg.TemplateFiles,
 		AlertmanagerConfig: definitions.GettableApiAlertingConfig{
 			Config: cfg.AlertmanagerConfig.Config,
 		},
+		ExtraConfigs: cfg.ExtraConfigs,
 	}
 
 	// First we encrypt the secure settings.
@@ -339,6 +351,82 @@ func (moa *MultiOrgAlertmanager) SaveAndApplyAlertmanagerConfiguration(ctx conte
 	return nil
 }
 
+// modifyAndApplyExtraConfiguration is a helper function that loads the current configuration,
+// applies a modification function to the ExtraConfigs, and saves the result.
+func (moa *MultiOrgAlertmanager) modifyAndApplyExtraConfiguration(
+	ctx context.Context,
+	org int64,
+	modifyFn func([]definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error),
+) error {
+	currentCfg, err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, org)
+	if err != nil {
+		return fmt.Errorf("failed to get current configuration: %w", err)
+	}
+
+	cfg, err := Load([]byte(currentCfg.AlertmanagerConfiguration))
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal current alertmanager configuration: %w", err)
+	}
+
+	cfg.ExtraConfigs, err = modifyFn(cfg.ExtraConfigs)
+	if err != nil {
+		return fmt.Errorf("failed to apply extra configuration: %w", err)
+	}
+
+	am, err := moa.AlertmanagerFor(org)
+	if err != nil {
+		// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
+		if !errors.Is(err, ErrAlertmanagerNotReady) {
+			return err
+		}
+	}
+
+	if err := am.SaveAndApplyConfig(ctx, cfg); err != nil {
+		moa.logger.Error("Unable to save and apply alertmanager configuration with extra config", "error", err, "org", org)
+		return AlertmanagerConfigRejectedError{err}
+	}
+
+	moa.logger.Info("Applied alertmanager configuration with extra config", "org", org)
+	return nil
+}
+
+// SaveAndApplyExtraConfiguration adds or replaces an ExtraConfiguration while preserving the main AlertmanagerConfig.
+func (moa *MultiOrgAlertmanager) SaveAndApplyExtraConfiguration(ctx context.Context, org int64, extraConfig definitions.ExtraConfiguration) error {
+	modifyFunc := func(configs []definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error) {
+		// for now we validate that after the update there will be just one extra config.
+		for _, c := range configs {
+			if c.Identifier != extraConfig.Identifier {
+				return nil, ErrAlertmanagerMultipleExtraConfigsUnsupported.Build(errutil.TemplateData{Public: map[string]interface{}{"Identifier": c.Identifier}})
+			}
+		}
+
+		return []definitions.ExtraConfiguration{extraConfig}, nil
+	}
+
+	err := moa.modifyAndApplyExtraConfiguration(ctx, org, modifyFunc)
+	if err != nil {
+		return err
+	}
+
+	moa.logger.Info("Applied alertmanager configuration with extra config", "org", org, "identifier", extraConfig.Identifier)
+	return nil
+}
+
+// DeleteAndApplyExtraConfiguration deletes an ExtraConfiguration by its identifier while preserving the main AlertmanagerConfig.
+func (moa *MultiOrgAlertmanager) DeleteAndApplyExtraConfiguration(ctx context.Context, org int64, identifier string) error {
+	modifyFunc := func(configs []definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error) {
+		filtered := make([]definitions.ExtraConfiguration, 0, len(configs))
+		for _, ec := range configs {
+			if ec.Identifier != identifier {
+				filtered = append(filtered, ec)
+			}
+		}
+		return filtered, nil
+	}
+
+	return moa.modifyAndApplyExtraConfiguration(ctx, org, modifyFunc)
+}
+
 // assignReceiverConfigsUIDs assigns missing UUIDs to receiver configs.
 func assignReceiverConfigsUIDs(c []*definitions.PostableApiReceiver) error {
 	seenUIDs := make(map[string]struct{})

pkg/services/ngalert/notifier/alertmanager_config_test.go

@@ -0,0 +1,204 @@
+package notifier
+
+import (
+	"context"
+	"testing"
+
+	amconfig "github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/pkg/labels"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+)
+
+func TestMultiOrgAlertmanager_SaveAndApplyExtraConfiguration(t *testing.T) {
+	orgID := int64(1)
+
+	t.Run("fails when organization does not exist", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		extraConfig := definitions.ExtraConfiguration{
+			Identifier: "test-config",
+			AlertmanagerConfig: `route:
+  receiver: test-receiver`,
+		}
+
+		err := mam.SaveAndApplyExtraConfiguration(ctx, 999, extraConfig)
+		require.Error(t, err)
+		require.ErrorContains(t, err, "failed to get current configuration")
+	})
+
+	t.Run("save new extra configuration", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		extraConfig := definitions.ExtraConfiguration{
+			Identifier:    "test-alertmanager-config",
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "prod"}},
+			TemplateFiles: map[string]string{"test.tmpl": "{{ define \"test\" }}Test{{ end }}"},
+			AlertmanagerConfig: `route:
+  receiver: test-receiver
+receivers:
+  - name: test-receiver`,
+		}
+
+		err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, extraConfig)
+		require.NoError(t, err)
+
+		gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
+		require.NoError(t, err)
+		require.Len(t, gettableConfig.ExtraConfigs, 1)
+		require.Equal(t, extraConfig.Identifier, gettableConfig.ExtraConfigs[0].Identifier)
+		require.Equal(t, extraConfig.TemplateFiles, gettableConfig.ExtraConfigs[0].TemplateFiles)
+
+		// Test that we can get the alertmanager config from raw storage
+		// We need to pass a decrypt function since the config is now encrypted
+		amConfig, err := gettableConfig.ExtraConfigs[0].GetAlertmanagerConfig()
+		require.NoError(t, err)
+		require.Equal(t, "test-receiver", amConfig.Route.Receiver)
+		require.Len(t, amConfig.Receivers, 1)
+		require.Equal(t, "test-receiver", amConfig.Receivers[0].Name)
+	})
+
+	t.Run("replace existing extra configuration with same identifier", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		identifier := "test-config"
+
+		// First add a configuration
+		originalConfig := definitions.ExtraConfiguration{
+			Identifier:    identifier,
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "original"}},
+			AlertmanagerConfig: `route:
+  receiver: original-receiver
+receivers:
+  - name: original-receiver`,
+		}
+
+		err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, originalConfig)
+		require.NoError(t, err)
+
+		// Now replace it
+		updatedConfig := definitions.ExtraConfiguration{
+			Identifier:    identifier,
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "updated"}},
+			TemplateFiles: map[string]string{"updated.tmpl": "{{ define \"updated\" }}Updated{{ end }}"},
+			AlertmanagerConfig: `route:
+  receiver: updated-receiver
+receivers:
+  - name: updated-receiver`,
+		}
+
+		err = mam.SaveAndApplyExtraConfiguration(ctx, orgID, updatedConfig)
+		require.NoError(t, err)
+
+		// Verify only one config exists with updated content
+		gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
+		require.NoError(t, err)
+		require.Len(t, gettableConfig.ExtraConfigs, 1)
+		require.Equal(t, identifier, gettableConfig.ExtraConfigs[0].Identifier)
+		require.Contains(t, gettableConfig.ExtraConfigs[0].TemplateFiles, "updated.tmpl")
+	})
+
+	t.Run("fail to create multiple extra configurations", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		firstConfig := definitions.ExtraConfiguration{
+			Identifier:    "first-config",
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "first"}},
+			AlertmanagerConfig: `{
+				"route": {
+					"receiver": "first-receiver"
+				},
+				"receivers": [
+					{
+						"name": "first-receiver"
+					}
+				]
+			}`,
+		}
+
+		err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, firstConfig)
+		require.NoError(t, err)
+
+		secondConfig := definitions.ExtraConfiguration{
+			Identifier:    "second-config",
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "second"}},
+			AlertmanagerConfig: `{
+				"route": {
+					"receiver": "second-receiver"
+				},
+				"receivers": [
+					{
+						"name": "second-receiver"
+					}
+				]
+			}`,
+		}
+
+		err = mam.SaveAndApplyExtraConfiguration(ctx, orgID, secondConfig)
+		require.Error(t, err)
+		require.ErrorContains(t, err, "multiple extra configurations are not supported")
+		require.ErrorContains(t, err, "first-config")
+	})
+}
+
+func TestMultiOrgAlertmanager_DeleteAndApplyExtraConfiguration(t *testing.T) {
+	orgID := int64(1)
+
+	t.Run("successfully delete existing extra configuration", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		identifier := "test-identifier"
+
+		extraConfig := definitions.ExtraConfiguration{
+			Identifier:    identifier,
+			MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "delete"}},
+			AlertmanagerConfig: `route:
+  receiver: test-receiver
+receivers:
+  - name: test-receiver`,
+		}
+
+		err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, extraConfig)
+		require.NoError(t, err)
+
+		gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
+		require.NoError(t, err)
+		require.Len(t, gettableConfig.ExtraConfigs, 1)
+
+		err = mam.DeleteAndApplyExtraConfiguration(ctx, orgID, identifier)
+		require.NoError(t, err)
+
+		gettableConfig, err = mam.GetAlertmanagerConfiguration(ctx, orgID, false)
+		require.NoError(t, err)
+		require.Len(t, gettableConfig.ExtraConfigs, 0)
+	})
+
+	t.Run("deletion of non-existent configuration", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+		require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
+
+		err := mam.DeleteAndApplyExtraConfiguration(ctx, orgID, "non-existent")
+		require.NoError(t, err)
+	})
+
+	t.Run("deletion in non-existent org fails", func(t *testing.T) {
+		mam := setupMam(t, nil)
+		ctx := context.Background()
+
+		err := mam.DeleteAndApplyExtraConfiguration(ctx, 999, "test-config")
+		require.Error(t, err)
+		require.ErrorContains(t, err, "failed to get current configuration")
+	})
+}

pkg/services/ngalert/notifier/alertmanager_test.go

@@ -2,15 +2,12 @@ package notifier
 
 import (
 	"context"
-	"net/url"
 	"testing"
 	"time"
 
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/client_golang/prometheus"
-	promcfg "github.com/prometheus/common/config"
-	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/db"
@@ -58,8 +55,9 @@ func setupAMTest(t *testing.T) *alertmanager {
 
 	orgID := 1
 	stateStore := NewFileStore(int64(orgID), kvStore)
+	crypto := NewCrypto(secretsService, s, l)
 
-	am, err := NewAlertmanager(context.Background(), 1, cfg, s, stateStore, &NilPeer{}, decryptFn, nil, m, featuremgmt.WithFeatures())
+	am, err := NewAlertmanager(context.Background(), 1, cfg, s, stateStore, &NilPeer{}, decryptFn, nil, m, featuremgmt.WithFeatures(), crypto)
 	require.NoError(t, err)
 	return am
 }
@@ -130,33 +128,17 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
 						TemplateFiles: map[string]string{
 							"mimir-template": "{{ define \"mimir.title\" }}Mimir Alert{{ end }}",
 						},
-						AlertmanagerConfig: definitions.PostableApiAlertingConfig{
+						AlertmanagerConfig: `route:
-							Config: definitions.Config{
+  receiver: mimir-webhook
-								Route: &definitions.Route{
+  group_by:
-									Receiver: "mimir-webhook",
+    - alertname
-									GroupBy:  []model.LabelName{"alertname", "cluster"},
+    - cluster
-								},
+receivers:
-							},
+  - name: mimir-webhook
-							Receivers: []*definitions.PostableApiReceiver{
+    webhook_configs:
-								{
+      - url: https://webhook.example.com/alerts
-									Receiver: config.Receiver{
+        send_resolved: true
-										Name: "mimir-webhook",
+        http_config: {}`,
-										WebhookConfigs: []*config.WebhookConfig{
-											{
-												URL: &config.SecretURL{
-													URL: &url.URL{
-														Scheme: "https",
-														Host:   "webhook.example.com",
-														Path:   "/alerts",
-													},
-												},
-												HTTPConfig: &promcfg.DefaultHTTPClientConfig,
-											},
-										},
-									},
-								},
-							},
-						},
 					},
 				},
 			},
@@ -170,13 +152,10 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
 					{
 						Identifier:    "", // invalid: empty identifier
 						MergeMatchers: config.Matchers{},
-						AlertmanagerConfig: definitions.PostableApiAlertingConfig{
+						AlertmanagerConfig: `route:
-							Config: definitions.Config{
+  receiver: test-receiver
-								Route: &definitions.Route{
+receivers:
-									Receiver: "test-receiver",
+  - name: test-receiver`,
-								},
-							},
-						},
 					},
 				},
 			},
@@ -190,15 +169,13 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
 			am := setupAMTest(t)
 			ctx := context.Background()
 
-			changed, err := am.applyConfig(ctx, tc.config, false)
+			err := am.SaveAndApplyConfig(ctx, tc.config)
 
 			if tc.expectedError != "" {
 				require.Error(t, err)
 				require.ErrorContains(t, err, tc.expectedError)
-				require.False(t, changed)
 			} else {
 				require.NoError(t, err)
-				require.True(t, changed)
 
 				templateDefs := tc.config.GetMergedTemplateDefinitions()
 				expectedTemplateCount := len(tc.config.TemplateFiles)

pkg/services/ngalert/notifier/crypto.go

@@ -18,6 +18,8 @@ import (
 type Crypto interface {
 	LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error
 	Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
+	EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
+	DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
 
 	getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error)
 	ProcessSecureSettings(ctx context.Context, orgId int64, recvs []*definitions.PostableApiReceiver) error
@@ -233,3 +235,34 @@ func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaRe
 func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
 	return c.secrets.Encrypt(ctx, payload, opt)
 }
+
+func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+	for i := range config.ExtraConfigs {
+		encryptedValue, err := c.secrets.Encrypt(ctx, []byte(config.ExtraConfigs[i].AlertmanagerConfig), secrets.WithoutScope())
+		if err != nil {
+			return fmt.Errorf("failed to encrypt extra configuration: %w", err)
+		}
+
+		config.ExtraConfigs[i].AlertmanagerConfig = base64.StdEncoding.EncodeToString(encryptedValue)
+	}
+
+	return nil
+}
+
+func (c *alertmanagerCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+	for i := range config.ExtraConfigs {
+		encryptedValue, err := base64.StdEncoding.DecodeString(config.ExtraConfigs[i].AlertmanagerConfig)
+		if err != nil {
+			return fmt.Errorf("failed to base64 decode extra configuration: %w", err)
+		}
+
+		decryptedValue, err := c.secrets.Decrypt(ctx, encryptedValue)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt extra configuration: %w", err)
+		}
+
+		config.ExtraConfigs[i].AlertmanagerConfig = string(decryptedValue)
+	}
+
+	return nil
+}

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -160,7 +160,7 @@ func NewMultiOrgAlertmanager(
 	moa.factory = func(ctx context.Context, orgID int64) (Alertmanager, error) {
 		m := metrics.NewAlertmanagerMetrics(moa.metrics.GetOrCreateOrgRegistry(orgID), l)
 		stateStore := NewFileStore(orgID, kvStore)
-		return NewAlertmanager(ctx, orgID, moa.settings, moa.configStore, stateStore, moa.peer, moa.decryptFn, moa.ns, m, featureManager)
+		return NewAlertmanager(ctx, orgID, moa.settings, moa.configStore, stateStore, moa.peer, moa.decryptFn, moa.ns, m, featureManager, moa.Crypto)
 	}
 
 	for _, opt := range opts {

public/api-merged.json

@@ -15355,6 +15355,26 @@
         }
       }
     },
+    "ExtraConfiguration": {
+      "type": "object",
+      "properties": {
+        "alertmanager_config": {
+          "type": "string"
+        },
+        "identifier": {
+          "type": "string"
+        },
+        "merge_matchers": {
+          "$ref": "#/definitions/Matchers"
+        },
+        "template_files": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "FailedUser": {
       "description": "FailedUser holds the information of an user that failed",
       "type": "object",
@@ -16304,6 +16324,12 @@
         "alertmanager_config": {
           "$ref": "#/definitions/GettableApiAlertingConfig"
         },
+        "extra_config": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ExtraConfiguration"
+          }
+        },
         "template_file_provenances": {
           "type": "object",
           "additionalProperties": {
@@ -18657,6 +18683,12 @@
         "alertmanager_config": {
           "$ref": "#/definitions/PostableApiAlertingConfig"
         },
+        "extra_config": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ExtraConfiguration"
+          }
+        },
         "template_files": {
           "type": "object",
           "additionalProperties": {
@@ -20041,8 +20073,8 @@
       }
     },
     "Route": {
-      "description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
       "type": "object",
+      "title": "A Route is a node that contains definitions of how to handle alerts.",
       "properties": {
         "active_time_intervals": {
           "type": "array",
@@ -20084,12 +20116,6 @@
             "type": "string"
           }
         },
-        "object_matchers": {
-          "$ref": "#/definitions/ObjectMatchers"
-        },
-        "provenance": {
-          "$ref": "#/definitions/Provenance"
-        },
         "receiver": {
           "type": "string"
         },

public/openapi3.json

@@ -5406,6 +5406,26 @@
         },
         "type": "object"
       },
+      "ExtraConfiguration": {
+        "properties": {
+          "alertmanager_config": {
+            "type": "string"
+          },
+          "identifier": {
+            "type": "string"
+          },
+          "merge_matchers": {
+            "$ref": "#/components/schemas/Matchers"
+          },
+          "template_files": {
+            "additionalProperties": {
+              "type": "string"
+            },
+            "type": "object"
+          }
+        },
+        "type": "object"
+      },
       "FailedUser": {
         "description": "FailedUser holds the information of an user that failed",
         "properties": {
@@ -6354,6 +6374,12 @@
           "alertmanager_config": {
             "$ref": "#/components/schemas/GettableApiAlertingConfig"
           },
+          "extra_config": {
+            "items": {
+              "$ref": "#/components/schemas/ExtraConfiguration"
+            },
+            "type": "array"
+          },
           "template_file_provenances": {
             "additionalProperties": {
               "$ref": "#/components/schemas/Provenance"
@@ -8707,6 +8733,12 @@
           "alertmanager_config": {
             "$ref": "#/components/schemas/PostableApiAlertingConfig"
           },
+          "extra_config": {
+            "items": {
+              "$ref": "#/components/schemas/ExtraConfiguration"
+            },
+            "type": "array"
+          },
           "template_files": {
             "additionalProperties": {
               "type": "string"
@@ -10092,7 +10124,6 @@
         "type": "object"
       },
       "Route": {
-        "description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
         "properties": {
           "active_time_intervals": {
             "items": {
@@ -10134,12 +10165,6 @@
             },
             "type": "array"
           },
-          "object_matchers": {
-            "$ref": "#/components/schemas/ObjectMatchers"
-          },
-          "provenance": {
-            "$ref": "#/components/schemas/Provenance"
-          },
           "receiver": {
             "type": "string"
           },
@@ -10153,6 +10178,7 @@
             "type": "array"
           }
         },
+        "title": "A Route is a node that contains definitions of how to handle alerts.",
         "type": "object"
       },
       "RouteExport": {