From 3584a8766d33bcaf3778fbd5ed32c54b0017dc0f Mon Sep 17 00:00:00 2001 From: gotjosh Date: Fri, 1 Dec 2023 14:22:59 +0000 Subject: [PATCH] [v10.2.x] Alerting: Only warm alert state cache if execute_alerts=true. (#78945) Alerting: Only warm alert state cache if execute_alerts=true. (#78895) * Alerting: Only warm alert state cache if execute_alerts=true. If the Grafana instance is not executing alerts, then Warm()-ing the state manager is wasteful and could lead to misleading rule status queries, as the status returned will be always based on the state loaded from the database at startup, and not the most recent evaluation state. * Move Warm() down to shared conditional. (cherry picked from commit 520c927931796cdeead8761813383fad275f6a44) Co-authored-by: Steve Simpson --- pkg/services/ngalert/ngalert.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index f73ddbfbc8a..553cb8ed120 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -318,9 +318,7 @@ func subscribeToFolderChanges(logger log.Logger, bus bus.Bus, dbStore api.RuleSt // Run starts the scheduler and Alertmanager. func (ng *AlertNG) Run(ctx context.Context) error { - ng.Log.Debug("Starting") - - ng.stateManager.Warm(ctx, ng.store) + ng.Log.Debug("Starting", "execute_alerts", ng.Cfg.UnifiedAlerting.ExecuteAlerts) children, subCtx := errgroup.WithContext(ctx) @@ -332,6 +330,17 @@ func (ng *AlertNG) Run(ctx context.Context) error { }) if ng.Cfg.UnifiedAlerting.ExecuteAlerts { + // Only Warm() the state manager if we are actually executing alerts. + // Doing so when we are not executing alerts is wasteful and could lead + // to misleading rule status queries, as the status returned will be + // always based on the state loaded from the database at startup, and + // not the most recent evaluation state. + // + // Also note that this runs synchronously to ensure state is loaded + // before rule evaluation begins, hence we use ctx and not subCtx. + // + ng.stateManager.Warm(ctx, ng.store) + children.Go(func() error { return ng.schedule.Run(subCtx) })