Remote Alertmanager: Add timeouts to the HTTP client (#105279)

* Remote Alertmanage: Add timeouts to the HTTP client

* code review suggestions

conf/defaults.ini

@@ -1577,6 +1577,9 @@ password =
 
 sync_interval = 5m
 
+# Timeout for the HTTP client. Default is 30 seconds.
+timeout = 30s
+
 #################################### Annotations #########################
 [annotations]
 # Configures the batch size for the annotation clean-up job. This setting is used for dashboard, API, and alert annotations.

pkg/services/ngalert/ngalert.go

@@ -195,6 +195,7 @@ func (ng *AlertNG) init() error {
 			ExternalURL:       ng.Cfg.AppURL,
 			SmtpFrom:          ng.Cfg.Smtp.FromAddress,
 			StaticHeaders:     ng.Cfg.Smtp.StaticHeaders,
+			Timeout:           ng.Cfg.UnifiedAlerting.RemoteAlertmanager.Timeout,
 		}
 		autogenFn := func(ctx context.Context, logger log.Logger, orgID int64, cfg *definitions.PostableApiAlertingConfig, skipInvalid bool) error {
 			return notifier.AddAutogenConfig(ctx, logger, ng.store, orgID, cfg, skipInvalid)

pkg/services/ngalert/remote/alertmanager.go

@@ -93,6 +93,9 @@ type AlertmanagerConfig struct {
 
 	// SyncInterval determines how often we should attempt to synchronize configuration.
 	SyncInterval time.Duration
+
+	// Timeout for the HTTP client.
+	Timeout time.Duration
 }
 
 func (cfg *AlertmanagerConfig) Validate() error {
@@ -141,6 +144,7 @@ func NewAlertmanager(ctx context.Context, cfg AlertmanagerConfig, store stateSto
 		TenantID: cfg.TenantID,
 		Password: cfg.BasicAuthPassword,
 		Logger:   logger,
+		Timeout:  cfg.Timeout,
 	}
 	amc, err := remoteClient.NewAlertmanager(amcCfg, metrics, tracer)
 	if err != nil {

pkg/services/ngalert/remote/client/alertmanager.go

@@ -24,6 +24,7 @@ type AlertmanagerConfig struct {
 	Password string
 	URL      *url.URL
 	Logger   log.Logger
+	Timeout  time.Duration
 }
 
 type Alertmanager struct {
@@ -34,12 +35,15 @@ type Alertmanager struct {
 }
 
 func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
-	// First, add the authentication middleware.
-	c := &http.Client{Transport: &MimirAuthRoundTripper{
-		TenantID: cfg.TenantID,
-		Password: cfg.Password,
-		Next:     httpclient.NewHTTPTransport(),
-	}}
+	// First, set up the http client.
+	c := &http.Client{
+		Transport: &MimirAuthRoundTripper{
+			TenantID: cfg.TenantID,
+			Password: cfg.Password,
+			Next:     httpclient.NewHTTPTransport(),
+		},
+		Timeout: cfg.Timeout,
+	}
 
 	tc := client.NewTimedClient(c, metrics.RequestLatency)
 	trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")

pkg/setting/setting_unified_alerting.go

@@ -47,6 +47,7 @@ const (
 `
 	alertingDefaultInitializationTimeout    = 30 * time.Second
 	evaluatorDefaultEvaluationTimeout       = 30 * time.Second
+	remoteAlertmanagerDefaultTimeout        = 30 * time.Second
 	schedulerDefaultAdminConfigPollInterval = time.Minute
 	schedulerDefaultExecuteAlerts           = true
 	schedulerDefaultMaxAttempts             = 3
@@ -152,6 +153,7 @@ type RemoteAlertmanagerSettings struct {
 	TenantID     string
 	Password     string
 	SyncInterval time.Duration
+	Timeout      time.Duration
 }
 
 type UnifiedAlertingScreenshotSettings struct {
@@ -396,6 +398,10 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	if err != nil {
 		return err
 	}
+	uaCfgRemoteAM.Timeout, err = gtime.ParseDuration(valueAsString(remoteAlertmanager, "timeout", (remoteAlertmanagerDefaultTimeout).String()))
+	if err != nil {
+		return err
+	}
 
 	uaCfg.RemoteAlertmanager = uaCfgRemoteAM