From 7a0ee7044bdd1fbd27871206a19a1b4285affe97 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Fri, 1 Dec 2023 14:23:08 +0000 Subject: [PATCH] [v10.1.x] Alerting: Only warm alert state cache if execute_alerts=true. (#78944) Alerting: Only warm alert state cache if execute_alerts=true. (#78895) * Alerting: Only warm alert state cache if execute_alerts=true. If the Grafana instance is not executing alerts, then Warm()-ing the state manager is wasteful and could lead to misleading rule status queries, as the status returned will be always based on the state loaded from the database at startup, and not the most recent evaluation state. * Move Warm() down to shared conditional. (cherry picked from commit 520c927931796cdeead8761813383fad275f6a44) Co-authored-by: Steve Simpson --- pkg/services/ngalert/ngalert.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 3b2f0844fa5..98a7cf204c6 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -313,8 +313,7 @@ func subscribeToFolderChanges(logger log.Logger, bus bus.Bus, dbStore api.RuleSt // Run starts the scheduler and Alertmanager. func (ng *AlertNG) Run(ctx context.Context) error { - ng.Log.Debug("Starting") - ng.stateManager.Warm(ctx, ng.store) + ng.Log.Debug("Starting", "execute_alerts", ng.Cfg.UnifiedAlerting.ExecuteAlerts) children, subCtx := errgroup.WithContext(ctx) @@ -330,6 +329,17 @@ func (ng *AlertNG) Run(ctx context.Context) error { }) if ng.Cfg.UnifiedAlerting.ExecuteAlerts { + // Only Warm() the state manager if we are actually executing alerts. + // Doing so when we are not executing alerts is wasteful and could lead + // to misleading rule status queries, as the status returned will be + // always based on the state loaded from the database at startup, and + // not the most recent evaluation state. + // + // Also note that this runs synchronously to ensure state is loaded + // before rule evaluation begins, hence we use ctx and not subCtx. + // + ng.stateManager.Warm(ctx, ng.store) + children.Go(func() error { return ng.schedule.Run(subCtx) })