From 361312bbd7f6e6276655d90903df7ea3a4071dae Mon Sep 17 00:00:00 2001 From: Santiago Date: Fri, 24 Jan 2025 16:26:57 +0100 Subject: [PATCH] Alerting: Expect 406s from the remote Alertmanager during the readiness check (#99507) * Alerting: Expect 406s from the remote Alertmanager during the readiness check * make it clear in the warning logs that we'll attempt to send the confgiuration/state without comparing in case of error pulling the current state/config --- pkg/services/ngalert/remote/alertmanager.go | 4 ++-- pkg/services/ngalert/remote/client/alertmanager.go | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 3d209aa85b9..82b222713ef 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -642,7 +642,7 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *apimodels. rc, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx) if err != nil { // Log the error and return true so we try to upload our config anyway. - am.log.Error("Unable to get the remote Alertmanager configuration for comparison", "err", err) + am.log.Warn("Unable to get the remote Alertmanager configuration for comparison, sending the configuration without comparing", "err", err) return true } @@ -669,7 +669,7 @@ func (am *Alertmanager) shouldSendState(ctx context.Context, state string) bool rs, err := am.mimirClient.GetGrafanaAlertmanagerState(ctx) if err != nil { // Log the error and return true so we try to upload our state anyway. - am.log.Error("Unable to get the remote Alertmanager state for comparison", "err", err) + am.log.Warn("Unable to get the remote Alertmanager state for comparison, sending the state without comparing", "err", err) return true } diff --git a/pkg/services/ngalert/remote/client/alertmanager.go b/pkg/services/ngalert/remote/client/alertmanager.go index 2ca7a9bebee..bc228884460 100644 --- a/pkg/services/ngalert/remote/client/alertmanager.go +++ b/pkg/services/ngalert/remote/client/alertmanager.go @@ -110,6 +110,14 @@ func (am *Alertmanager) IsReadyWithBackoff(ctx context.Context) (bool, error) { if status != http.StatusOK { if status >= 400 && status < 500 { + if status == http.StatusNotAcceptable { + // Mimir returns a 406 when the Alertmanager for the tenant is not running. + // This is expected if the Grafana Alertmanager configuration is default or not promoted. + // We can still use the endpoints to store and retrieve configuration/state. + am.logger.Debug("Remote Alertmanager not initialized for tenant", "attempt", attempts, "status", status) + return true, nil + } + am.logger.Debug("Ready check failed with non-retriable status code", "attempt", attempts, "status", status) return false, fmt.Errorf("ready check failed with non-retriable status code %d", status) }