From 6c3d89f390369b03ad4b3e8ae541c10cfa957d6e Mon Sep 17 00:00:00 2001
From: Santiago <santiagohernandez.1997@gmail.com>
Date: Tue, 13 May 2025 13:25:56 +0200
Subject: [PATCH] Remote Alertmanager: Add timeouts to the HTTP client
 (#105279)

* Remote Alertmanage: Add timeouts to the HTTP client

* code review suggestions
---
 conf/defaults.ini                                |  3 +++
 pkg/services/ngalert/ngalert.go                  |  1 +
 pkg/services/ngalert/remote/alertmanager.go      |  4 ++++
 .../ngalert/remote/client/alertmanager.go        | 16 ++++++++++------
 pkg/setting/setting_unified_alerting.go          |  6 ++++++
 5 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/conf/defaults.ini b/conf/defaults.ini
index 97e37426cd6..46bfdf8f540 100644
--- a/conf/defaults.ini
+++ b/conf/defaults.ini
@@ -1577,6 +1577,9 @@ password =
 
 sync_interval = 5m
 
+# Timeout for the HTTP client. Default is 30 seconds.
+timeout = 30s
+
 #################################### Annotations #########################
 [annotations]
 # Configures the batch size for the annotation clean-up job. This setting is used for dashboard, API, and alert annotations.
diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go
index 7e1bfe53a12..bf764a3647b 100644
--- a/pkg/services/ngalert/ngalert.go
+++ b/pkg/services/ngalert/ngalert.go
@@ -195,6 +195,7 @@ func (ng *AlertNG) init() error {
 			ExternalURL:       ng.Cfg.AppURL,
 			SmtpFrom:          ng.Cfg.Smtp.FromAddress,
 			StaticHeaders:     ng.Cfg.Smtp.StaticHeaders,
+			Timeout:           ng.Cfg.UnifiedAlerting.RemoteAlertmanager.Timeout,
 		}
 		autogenFn := func(ctx context.Context, logger log.Logger, orgID int64, cfg *definitions.PostableApiAlertingConfig, skipInvalid bool) error {
 			return notifier.AddAutogenConfig(ctx, logger, ng.store, orgID, cfg, skipInvalid)
diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go
index 5788506eb4c..8eadbdb446e 100644
--- a/pkg/services/ngalert/remote/alertmanager.go
+++ b/pkg/services/ngalert/remote/alertmanager.go
@@ -93,6 +93,9 @@ type AlertmanagerConfig struct {
 
 	// SyncInterval determines how often we should attempt to synchronize configuration.
 	SyncInterval time.Duration
+
+	// Timeout for the HTTP client.
+	Timeout time.Duration
 }
 
 func (cfg *AlertmanagerConfig) Validate() error {
@@ -141,6 +144,7 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
 		TenantID: cfg.TenantID,
 		Password: cfg.BasicAuthPassword,
 		Logger:   logger,
+		Timeout:  cfg.Timeout,
 	}
 	amc, err := remoteClient.NewAlertmanager(amcCfg, metrics, tracer)
 	if err != nil {
diff --git a/pkg/services/ngalert/remote/client/alertmanager.go b/pkg/services/ngalert/remote/client/alertmanager.go
index f4d6c3ee033..52739aab594 100644
--- a/pkg/services/ngalert/remote/client/alertmanager.go
+++ b/pkg/services/ngalert/remote/client/alertmanager.go
@@ -24,6 +24,7 @@ type AlertmanagerConfig struct {
 	Password string
 	URL      *url.URL
 	Logger   log.Logger
+	Timeout  time.Duration
 }
 
 type Alertmanager struct {
@@ -34,12 +35,15 @@ type Alertmanager struct {
 }
 
 func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
-	// First, add the authentication middleware.
-	c := &http.Client{Transport: &MimirAuthRoundTripper{
-		TenantID: cfg.TenantID,
-		Password: cfg.Password,
-		Next:     httpclient.NewHTTPTransport(),
-	}}
+	// First, set up the http client.
+	c := &http.Client{
+		Transport: &MimirAuthRoundTripper{
+			TenantID: cfg.TenantID,
+			Password: cfg.Password,
+			Next:     httpclient.NewHTTPTransport(),
+		},
+		Timeout: cfg.Timeout,
+	}
 
 	tc := client.NewTimedClient(c, metrics.RequestLatency)
 	trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go
index b22484dbd69..866c269dc4d 100644
--- a/pkg/setting/setting_unified_alerting.go
+++ b/pkg/setting/setting_unified_alerting.go
@@ -47,6 +47,7 @@ const (
 `
 	alertingDefaultInitializationTimeout    = 30 * time.Second
 	evaluatorDefaultEvaluationTimeout       = 30 * time.Second
+	remoteAlertmanagerDefaultTimeout        = 30 * time.Second
 	schedulerDefaultAdminConfigPollInterval = time.Minute
 	schedulerDefaultExecuteAlerts           = true
 	schedulerDefaultMaxAttempts             = 3
@@ -152,6 +153,7 @@ type RemoteAlertmanagerSettings struct {
 	TenantID     string
 	Password     string
 	SyncInterval time.Duration
+	Timeout      time.Duration
 }
 
 type UnifiedAlertingScreenshotSettings struct {
@@ -396,6 +398,10 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	if err != nil {
 		return err
 	}
+	uaCfgRemoteAM.Timeout, err = gtime.ParseDuration(valueAsString(remoteAlertmanager, "timeout", (remoteAlertmanagerDefaultTimeout).String()))
+	if err != nil {
+		return err
+	}
 
 	uaCfg.RemoteAlertmanager = uaCfgRemoteAM