From b6b8554c5ea215aa36cb533a4093468f6947afb7 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Fri, 1 Dec 2023 14:23:17 +0000 Subject: [PATCH] [v10.0.x] Alerting: Only warm alert state cache if execute_alerts=true. (#78943) Alerting: Only warm alert state cache if execute_alerts=true. (#78895) * Alerting: Only warm alert state cache if execute_alerts=true. If the Grafana instance is not executing alerts, then Warm()-ing the state manager is wasteful and could lead to misleading rule status queries, as the status returned will be always based on the state loaded from the database at startup, and not the most recent evaluation state. * Move Warm() down to shared conditional. (cherry picked from commit 520c927931796cdeead8761813383fad275f6a44) Co-authored-by: Steve Simpson --- pkg/services/ngalert/ngalert.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 6a758aa1381..b608ff0516a 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -320,8 +320,7 @@ func subscribeToFolderChanges(logger log.Logger, bus bus.Bus, dbStore api.RuleSt // Run starts the scheduler and Alertmanager. func (ng *AlertNG) Run(ctx context.Context) error { - ng.Log.Debug("Starting") - ng.stateManager.Warm(ctx, ng.store) + ng.Log.Debug("Starting", "execute_alerts", ng.Cfg.UnifiedAlerting.ExecuteAlerts) children, subCtx := errgroup.WithContext(ctx) @@ -337,6 +336,17 @@ func (ng *AlertNG) Run(ctx context.Context) error { }) if ng.Cfg.UnifiedAlerting.ExecuteAlerts { + // Only Warm() the state manager if we are actually executing alerts. + // Doing so when we are not executing alerts is wasteful and could lead + // to misleading rule status queries, as the status returned will be + // always based on the state loaded from the database at startup, and + // not the most recent evaluation state. + // + // Also note that this runs synchronously to ensure state is loaded + // before rule evaluation begins, hence we use ctx and not subCtx. + // + ng.stateManager.Warm(ctx, ng.store) + children.Go(func() error { return ng.schedule.Run(subCtx) })