Alerting: Add support for client certificate authentication and TLS options to External Alertmanager (#115716)

* add support for skip TLS verify

* extract constructor for ExternalAMcfg and tests

* extract constructor for AlertmanagerConfig and tests

* add support for client cert auth

(cherry picked from commit 7ba2c559c4)

pkg/services/ngalert/sender/router.go

@@ -249,34 +249,67 @@ func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]ExternalAMcf
 		if !ds.JsonData.Get(definitions.HandleGrafanaManagedAlerts).MustBool(false) {
 			continue
 		}
-		amURL, err := d.buildExternalURL(ds)
+
+		cfg, err := d.datasourceToExternalAMcfg(ds)
 		if err != nil {
-			d.logger.Error("Failed to build external alertmanager URL",
-				"org", ds.OrgID,
-				"uid", ds.UID,
-				"error", err)
-			continue
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
-		headers, err := d.datasourceService.CustomHeaders(ctx, ds)
-		cancel()
-		if err != nil {
-			d.logger.Error("Failed to get headers for external alertmanager",
+			d.logger.Error("Failed to convert datasource to external alertmanager config",
 				"org", ds.OrgID,
 				"uid", ds.UID,
 				"error", err)
 			continue
 		}
 
-		alertmanagers = append(alertmanagers, ExternalAMcfg{
-			URL:     amURL,
-			Headers: headers,
-		})
+		alertmanagers = append(alertmanagers, cfg)
 	}
 
 	return alertmanagers, nil
 }
 
+// datasourceToExternalAMcfg converts a datasource to an ExternalAMcfg.
+func (d *AlertsRouter) datasourceToExternalAMcfg(ds *datasources.DataSource) (ExternalAMcfg, error) {
+	amURL, err := d.buildExternalURL(ds)
+	if err != nil {
+		return ExternalAMcfg{}, fmt.Errorf("failed to build external alertmanager URL: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	headers, err := d.datasourceService.CustomHeaders(ctx, ds)
+	cancel()
+	if err != nil {
+		return ExternalAMcfg{}, fmt.Errorf("failed to get custom headers: %w", err)
+	}
+
+	insecureSkipVerify := false
+
+	var tlsAuthEnabled bool
+	if ds.JsonData != nil {
+		insecureSkipVerify = ds.JsonData.Get("tlsSkipVerify").MustBool(false)
+		tlsAuthEnabled = ds.JsonData.Get("tlsAuth").MustBool(false)
+	}
+
+	var tlsClientCert, tlsClientKey string
+	if tlsAuthEnabled {
+		if ds.SecureJsonData == nil {
+			return ExternalAMcfg{}, errors.New("tlsAuth is enabled but TLS client certificate and key are not configured")
+		}
+
+		tlsClientKey = d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "tlsClientKey", "")
+		tlsClientCert = d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "tlsClientCert", "")
+
+		if tlsClientCert == "" || tlsClientKey == "" {
+			return ExternalAMcfg{}, errors.New("tlsAuth is enabled but TLS client certificate or key is empty")
+		}
+	}
+
+	return ExternalAMcfg{
+		URL:                amURL,
+		Headers:            headers,
+		InsecureSkipVerify: insecureSkipVerify,
+		TLSClientCert:      tlsClientCert,
+		TLSClientKey:       tlsClientKey,
+	}, nil
+}
+
 func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, error) {
 	// We re-use the same parsing logic as the datasource to make sure it matches whatever output the user received
 	// when doing the healthcheck.

pkg/services/ngalert/sender/router_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"testing"
 	"time"
@@ -744,3 +745,296 @@ func TestAlertManagers_buildRedactedAMs(t *testing.T) {
 		})
 	}
 }
+
+func TestDatasourceToExternalAMcfg(t *testing.T) {
+	tests := []struct {
+		name        string
+		datasource  *datasources.DataSource
+		expected    ExternalAMcfg
+		expectError bool
+	}{
+		{
+			name: "datasource with tlsSkipVerify enabled",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsSkipVerify": true,
+				}),
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://localhost:9093",
+				InsecureSkipVerify: true,
+			},
+		},
+		{
+			name: "datasource with tlsSkipVerify disabled",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsSkipVerify": false,
+				}),
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://localhost:9093",
+				InsecureSkipVerify: false,
+			},
+		},
+		{
+			name: "datasource without tlsSkipVerify (defaults to false)",
+			datasource: &datasources.DataSource{
+				URL:      "https://localhost:9093",
+				OrgID:    1,
+				Type:     datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{}),
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://localhost:9093",
+				InsecureSkipVerify: false,
+			},
+		},
+		{
+			name: "mimir datasource with tlsSkipVerify",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"implementation": "mimir",
+					"tlsSkipVerify":  true,
+				}),
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://localhost:9093/alertmanager",
+				InsecureSkipVerify: true,
+			},
+		},
+		{
+			name: "datasource with basic auth and tlsSkipVerify",
+			datasource: &datasources.DataSource{
+				URL:           "https://localhost:9093",
+				OrgID:         1,
+				Type:          datasources.DS_ALERTMANAGER,
+				BasicAuth:     true,
+				BasicAuthUser: "user",
+				SecureJsonData: map[string][]byte{
+					"basicAuthPassword": []byte("password"),
+				},
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsSkipVerify": true,
+				}),
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://user:password@localhost:9093",
+				InsecureSkipVerify: true,
+			},
+		},
+		{
+			name: "datasource with TLS client auth",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsAuth": true,
+				}),
+				SecureJsonData: map[string][]byte{
+					"tlsClientCert": []byte("client-cert-content"),
+					"tlsClientKey":  []byte("client-key-content"),
+				},
+			},
+			expected: ExternalAMcfg{
+				URL:           "https://localhost:9093",
+				TLSClientCert: "client-cert-content",
+				TLSClientKey:  "client-key-content",
+			},
+		},
+		{
+			name: "datasource with TLS client auth and skip verify",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsSkipVerify": true,
+					"tlsAuth":       true,
+				}),
+				SecureJsonData: map[string][]byte{
+					"tlsClientCert": []byte("client-cert-content"),
+					"tlsClientKey":  []byte("client-key-content"),
+				},
+			},
+			expected: ExternalAMcfg{
+				URL:                "https://localhost:9093",
+				InsecureSkipVerify: true,
+				TLSClientCert:      "client-cert-content",
+				TLSClientKey:       "client-key-content",
+			},
+		},
+		{
+			name: "tlsAuth enabled but SecureJsonData is nil - should error",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsAuth": true,
+				}),
+				SecureJsonData: nil,
+			},
+			expectError: true,
+		},
+		{
+			name: "tlsAuth enabled but tlsClientCert is empty - should error",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsAuth": true,
+				}),
+				SecureJsonData: map[string][]byte{
+					"tlsClientKey": []byte("client-key-content"),
+				},
+			},
+			expectError: true,
+		},
+		{
+			name: "tlsAuth enabled but tlsClientKey is empty - should error",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsAuth": true,
+				}),
+				SecureJsonData: map[string][]byte{
+					"tlsClientCert": []byte("client-cert-content"),
+				},
+			},
+			expectError: true,
+		},
+		{
+			name: "tlsAuth enabled but both cert and key are empty - should error",
+			datasource: &datasources.DataSource{
+				URL:   "https://localhost:9093",
+				OrgID: 1,
+				Type:  datasources.DS_ALERTMANAGER,
+				JsonData: simplejson.NewFromAny(map[string]any{
+					"tlsAuth": true,
+				}),
+				SecureJsonData: map[string][]byte{},
+			},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			router := &AlertsRouter{
+				logger:            log.New("test"),
+				datasourceService: &fake_ds.FakeDataSourceService{},
+				secretService:     fake_secrets.NewFakeSecretsService(),
+			}
+
+			cfg, err := router.datasourceToExternalAMcfg(tt.datasource)
+
+			if tt.expectError {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tt.expected, cfg)
+		})
+	}
+}
+
+func TestExternalAMcfg_SHA256(t *testing.T) {
+	// Golden config with all fields populated
+	goldenCfg := ExternalAMcfg{
+		URL: "https://localhost:9093",
+		Headers: http.Header{
+			"X-Custom-Header": []string{"value1"},
+			"Authorization":   []string{"Bearer token"},
+		},
+		Timeout:            30 * time.Second,
+		InsecureSkipVerify: true,
+		TLSClientCert:      "client-cert-content",
+		TLSClientKey:       "client-key-content",
+	}
+	goldenHash := goldenCfg.SHA256()
+
+	tests := []struct {
+		name         string
+		mutateFn     func(ExternalAMcfg) ExternalAMcfg
+		shouldDiffer bool
+	}{
+		{
+			name: "mutate URL - hash should change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.URL = "https://different-host:9093"
+				return cfg
+			},
+			shouldDiffer: true,
+		},
+		{
+			name: "mutate Headers - hash should change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.Headers = http.Header{
+					"X-Different-Header": []string{"different-value"},
+				}
+				return cfg
+			},
+			shouldDiffer: true,
+		},
+		{
+			name: "mutate Timeout - hash should NOT change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.Timeout = 60 * time.Second
+				return cfg
+			},
+			shouldDiffer: false,
+		},
+		{
+			name: "mutate InsecureSkipVerify - hash should change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.InsecureSkipVerify = false
+				return cfg
+			},
+			shouldDiffer: true,
+		},
+		{
+			name: "mutate TLSClientCert - hash should change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.TLSClientCert = "different-cert"
+				return cfg
+			},
+			shouldDiffer: true,
+		},
+		{
+			name: "mutate TLSClientKey - hash should change",
+			mutateFn: func(cfg ExternalAMcfg) ExternalAMcfg {
+				cfg.TLSClientKey = "different-key"
+				return cfg
+			},
+			shouldDiffer: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mutatedCfg := tt.mutateFn(goldenCfg)
+			mutatedHash := mutatedCfg.SHA256()
+
+			if tt.shouldDiffer {
+				require.NotEqual(t, goldenHash, mutatedHash, "Expected hash to change after mutation")
+			} else {
+				require.Equal(t, goldenHash, mutatedHash, "Expected hash to remain the same after mutation")
+			}
+		})
+	}
+}

pkg/services/ngalert/sender/sender.go

@@ -47,6 +47,12 @@ type ExternalAMcfg struct {
 	URL     string
 	Headers http.Header
 	Timeout time.Duration
+	// InsecureSkipVerify determines whether the server's TLS certificate should be verified.
+	InsecureSkipVerify bool
+	// TLSClientCert specifies the TLS client certificate used for secure communication.
+	TLSClientCert string
+	// TLSClientKey specifies the private key associated with the TLS client certificate for secure communication.
+	TLSClientKey string
 }
 
 type ExternalAMOptions struct {
@@ -94,7 +100,17 @@ func WithMaxBatchSize(size int) Option {
 }
 
 func (cfg *ExternalAMcfg) SHA256() string {
-	return asSHA256([]string{cfg.headerString(), cfg.URL})
+	skipVerify := "false"
+	if cfg.InsecureSkipVerify {
+		skipVerify = "true"
+	}
+	return asSHA256([]string{
+		cfg.headerString(),
+		cfg.URL,
+		skipVerify,
+		cfg.TLSClientCert,
+		cfg.TLSClientKey,
+	})
 }
 
 // headersString transforms all the headers in a sorted way as a
@@ -250,48 +266,17 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str
 	amConfigs := make([]*config.AlertmanagerConfig, 0, len(alertmanagers))
 	headers := map[string]http.Header{}
 	for i, am := range alertmanagers {
-		u, err := url.Parse(am.URL)
+		amConfig, err := externalAMcfgToAlertmanagerConfig(am)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		sdConfig := discovery.Configs{
-			discovery.StaticConfig{
-				{
-					Targets: []model.LabelSet{{model.AddressLabel: model.LabelValue(u.Host)}},
-				},
-			},
-		}
-
-		timeout := am.Timeout
-		if timeout == 0 {
-			timeout = defaultTimeout
-		}
-
-		amConfig := &config.AlertmanagerConfig{
-			APIVersion:              config.AlertmanagerAPIVersionV2,
-			Scheme:                  u.Scheme,
-			PathPrefix:              u.Path,
-			Timeout:                 model.Duration(timeout),
-			ServiceDiscoveryConfigs: sdConfig,
-		}
-
 		if am.Headers != nil {
 			// The key has the same format as the AlertmanagerConfigs.ToMap() would generate
 			// so we can use it later on when working with the alertmanager config map.
 			headers[fmt.Sprintf("config-%d", i)] = am.Headers
 		}
 
-		// Check the URL for basic authentication information first
-		if u.User != nil {
-			amConfig.HTTPClientConfig.BasicAuth = &common_config.BasicAuth{
-				Username: u.User.Username(),
-			}
-
-			if password, isSet := u.User.Password(); isSet {
-				amConfig.HTTPClientConfig.BasicAuth.Password = common_config.Secret(password)
-			}
-		}
 		amConfigs = append(amConfigs, amConfig)
 	}
 
@@ -304,6 +289,62 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str
 	return notifierConfig, headers, nil
 }
 
+// externalAMcfgToAlertmanagerConfig converts an ExternalAMcfg to a Prometheus AlertmanagerConfig.
+func externalAMcfgToAlertmanagerConfig(am ExternalAMcfg) (*config.AlertmanagerConfig, error) {
+	u, err := url.Parse(am.URL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse alertmanager URL: %w", err)
+	}
+
+	sdConfig := discovery.Configs{
+		discovery.StaticConfig{
+			{
+				Targets: []model.LabelSet{{model.AddressLabel: model.LabelValue(u.Host)}},
+			},
+		},
+	}
+
+	timeout := am.Timeout
+	if timeout == 0 {
+		timeout = defaultTimeout
+	}
+
+	amConfig := &config.AlertmanagerConfig{
+		APIVersion:              config.AlertmanagerAPIVersionV2,
+		Scheme:                  u.Scheme,
+		PathPrefix:              u.Path,
+		Timeout:                 model.Duration(timeout),
+		ServiceDiscoveryConfigs: sdConfig,
+	}
+
+	// Check the URL for basic authentication information first
+	if u.User != nil {
+		amConfig.HTTPClientConfig.BasicAuth = &common_config.BasicAuth{
+			Username: u.User.Username(),
+		}
+
+		if password, isSet := u.User.Password(); isSet {
+			amConfig.HTTPClientConfig.BasicAuth.Password = common_config.Secret(password)
+		}
+	}
+
+	// Validate that if TLS client cert is provided, key must also be provided (and vice versa)
+	if (am.TLSClientCert != "" && am.TLSClientKey == "") || (am.TLSClientCert == "" && am.TLSClientKey != "") {
+		return nil, fmt.Errorf("TLS client certificate and key must both be provided or both be empty")
+	}
+
+	// Set TLS configuration if any TLS options are provided
+	if am.InsecureSkipVerify || am.TLSClientCert != "" {
+		amConfig.HTTPClientConfig.TLSConfig = common_config.TLSConfig{
+			InsecureSkipVerify: am.InsecureSkipVerify,
+			Cert:               am.TLSClientCert,
+			Key:                common_config.Secret(am.TLSClientKey),
+		}
+	}
+
+	return amConfig, nil
+}
+
 func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *Alert {
 	// Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys.
 	return &Alert{

pkg/services/ngalert/sender/sender_test.go

@@ -3,9 +3,14 @@ package sender
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/client_golang/prometheus"
+	common_config "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/require"
 
@@ -227,3 +232,238 @@ func TestWithUTF8Labels(t *testing.T) {
 		require.Equal(t, "fire", result.Labels.Get("_0x1f525"))
 	})
 }
+
+func TestExternalAMcfgToAlertmanagerConfig(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         ExternalAMcfg
+		expected    *config.AlertmanagerConfig
+		expectError bool
+	}{
+		{
+			name: "basic configuration without TLS skip verify",
+			cfg: ExternalAMcfg{
+				URL:                "https://alertmanager.example.com:9093/alertmanager",
+				InsecureSkipVerify: false,
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "/alertmanager",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "configuration with TLS skip verify enabled",
+			cfg: ExternalAMcfg{
+				URL:                "https://alertmanager.example.com:9093",
+				InsecureSkipVerify: true,
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+				HTTPClientConfig: common_config.HTTPClientConfig{
+					TLSConfig: common_config.TLSConfig{
+						InsecureSkipVerify: true,
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "configuration with basic auth in URL",
+			cfg: ExternalAMcfg{
+				URL:                "https://user:password@alertmanager.example.com:9093",
+				InsecureSkipVerify: false,
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+				HTTPClientConfig: common_config.HTTPClientConfig{
+					BasicAuth: &common_config.BasicAuth{
+						Username: "user",
+						Password: "password",
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "configuration with basic auth and TLS skip verify",
+			cfg: ExternalAMcfg{
+				URL:                "https://user:password@alertmanager.example.com:9093",
+				InsecureSkipVerify: true,
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+				HTTPClientConfig: common_config.HTTPClientConfig{
+					BasicAuth: &common_config.BasicAuth{
+						Username: "user",
+						Password: "password",
+					},
+					TLSConfig: common_config.TLSConfig{
+						InsecureSkipVerify: true,
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "configuration with custom timeout",
+			cfg: ExternalAMcfg{
+				URL:                "https://alertmanager.example.com:9093",
+				Timeout:            30 * time.Second,
+				InsecureSkipVerify: false,
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(30 * time.Second),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "invalid URL should return error",
+			cfg: ExternalAMcfg{
+				URL:                "://invalid-url",
+				InsecureSkipVerify: false,
+			},
+			expected:    nil,
+			expectError: true,
+		},
+		{
+			name: "configuration with TLS client auth",
+			cfg: ExternalAMcfg{
+				URL:           "https://alertmanager.example.com:9093",
+				TLSClientCert: "client-cert-content",
+				TLSClientKey:  "client-key-content",
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+				HTTPClientConfig: common_config.HTTPClientConfig{
+					TLSConfig: common_config.TLSConfig{
+						Cert: "client-cert-content",
+						Key:  "client-key-content",
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "configuration with TLS client auth and skip verify",
+			cfg: ExternalAMcfg{
+				URL:                "https://alertmanager.example.com:9093",
+				InsecureSkipVerify: true,
+				TLSClientCert:      "client-cert-content",
+				TLSClientKey:       "client-key-content",
+			},
+			expected: &config.AlertmanagerConfig{
+				APIVersion: config.AlertmanagerAPIVersionV2,
+				Scheme:     "https",
+				PathPrefix: "",
+				Timeout:    model.Duration(defaultTimeout),
+				ServiceDiscoveryConfigs: discovery.Configs{
+					discovery.StaticConfig{
+						{
+							Targets: []model.LabelSet{{model.AddressLabel: "alertmanager.example.com:9093"}},
+						},
+					},
+				},
+				HTTPClientConfig: common_config.HTTPClientConfig{
+					TLSConfig: common_config.TLSConfig{
+						InsecureSkipVerify: true,
+						Cert:               "client-cert-content",
+						Key:                "client-key-content",
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "TLS client cert provided but key missing - should error",
+			cfg: ExternalAMcfg{
+				URL:           "https://alertmanager.example.com:9093",
+				TLSClientCert: "client-cert-content",
+			},
+			expected:    nil,
+			expectError: true,
+		},
+		{
+			name: "TLS client key provided but cert missing - should error",
+			cfg: ExternalAMcfg{
+				URL:          "https://alertmanager.example.com:9093",
+				TLSClientKey: "client-key-content",
+			},
+			expected:    nil,
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			amConfig, err := externalAMcfgToAlertmanagerConfig(tt.cfg)
+
+			if tt.expectError {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tt.expected, amConfig)
+		})
+	}
+}