public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support retry with backoff in alert rule evaluation (#99710)

Browse Source
This commit is contained in:
Alexander Akhmetov
2025-09-04 13:56:03 +02:00
committed by GitHub
parent 8052ecb3ba
commit 100528e274
13 changed files with 512 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conf/defaults.ini
+17 -2
View File
@@ -1412,10 +1412,25 @@ execute_alerts = true
# The timeout string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
evaluation_timeout = 30s
# Number of times we'll attempt to evaluate an alert rule before giving up on that evaluation. The default value is 3.
# Total number of evaluation attempts for an alert rule before giving up (including the initial attempt). The default value is 3.
# The retry mechanism will stop if this number is reached or if the rule's evaluation interval is exceeded.
# NOTE: For rules with short evaluation intervals, it's recommended to keep this value low and ensure that
# retry delays are shorter than the rule's evaluation interval to avoid resource contention.
max_attempts = 3
# Minimum interval to enforce between rule evaluations. Rules will be adjusted if they are less than this value or if they are not multiple of the scheduler interval (10s). Higher values can help with resource management as we'll schedule fewer evaluations over time.
# The initial delay before retrying a failed alert evaluation. This is the starting point for exponential backoff.
initial_retry_delay = 1s
# The maximum delay between retries during exponential backoff. Once this delay is reached, all subsequent retries will use this fixed interval.
# For optimal performance, ensure the total time of all retries is less than the rule's evaluation interval to prevent retry attempts from overlapping with scheduled evaluations.
max_retry_delay = 4s
# The randomization factor for exponential backoff retries. This adds jitter to retry delays to prevent thundering herd problems when multiple rules fail simultaneously.
# Value must be between 0 and 1. With factor F, the actual delay will be randomly chosen
# from [current_delay*(1-F), current_delay*(1+F)] where current_delay grows exponentially.
# Default is 0.1.
randomization_factor = 0.1
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
min_interval = 10s
conf/sample.ini
+17 -2
View File
@@ -1389,10 +1389,25 @@
# The timeout string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
;evaluation_timeout = 30s
# Number of times we'll attempt to evaluate an alert rule before giving up on that evaluation. The default value is 3.
# Total number of evaluation attempts for an alert rule before giving up (including the initial attempt). The default value is 3.
# The retry mechanism will stop if this number is reached or if the rule's evaluation interval is exceeded.
# NOTE: For rules with short evaluation intervals, it's recommended to keep this value low and ensure that
# retry delays are shorter than the rule's evaluation interval to avoid resource contention.
;max_attempts = 3
# Minimum interval to enforce between rule evaluations. Rules will be adjusted if they are less than this value or if they are not multiple of the scheduler interval (10s). Higher values can help with resource management as we'll schedule fewer evaluations over time.
# The initial delay before retrying a failed alert evaluation. This is the starting point for exponential backoff.
;initial_retry_delay = 1s
# The maximum delay between retries during exponential backoff. Once this delay is reached, all subsequent retries will use this fixed interval.
# For optimal performance, ensure the total time of all retries is less than the rule's evaluation interval to prevent retry attempts from overlapping with scheduled evaluations.
;max_retry_delay = 4s
# The randomization factor for exponential backoff retries. This adds jitter to retry delays to prevent thundering herd problems when multiple rules fail simultaneously.
# Value must be between 0 and 1. With factor F, the actual delay will be randomly chosen
# from [current_delay*(1-F), current_delay*(1+F)] where current_delay grows exponentially.
# Default is 0.1.
;randomization_factor = 0.1
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
;min_interval = 10s
go.mod
+1 -1
View File
@@ -366,7 +366,7 @@ require (
github.com/buger/jsonparser v1.1.1 // indirect
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 // indirect
github.com/caio/go-tdigest v3.1.0+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // @grafana/alerting-backend
github.com/cenkalti/backoff/v5 v5.0.2 // indirect
github.com/centrifugal/protocol v0.16.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
pkg/services/ngalert/ngalert.go
+6 -1
View File
@@ -321,7 +321,12 @@ func (ng *AlertNG) init() error {
ng.RecordingWriter = recordingWriter
schedCfg := schedule.SchedulerCfg{
MaxAttempts: ng.Cfg.UnifiedAlerting.MaxAttempts,
RetryConfig: schedule.RetryConfig{
MaxAttempts: ng.Cfg.UnifiedAlerting.MaxAttempts,
InitialRetryDelay: ng.Cfg.UnifiedAlerting.InitialRetryDelay,
MaxRetryDelay: ng.Cfg.UnifiedAlerting.MaxRetryDelay,
RandomizationFactor: ng.Cfg.UnifiedAlerting.RandomizationFactor,
},
C: clk,
BaseInterval: ng.Cfg.UnifiedAlerting.BaseInterval,
MinRuleInterval: ng.Cfg.UnifiedAlerting.MinInterval,
pkg/services/ngalert/schedule/alert_rule.go
+24 -11
View File
@@ -56,7 +56,7 @@ func (f ruleFactoryFunc) new(ctx context.Context, rule *ngmodels.AlertRule) Rule
func newRuleFactory(
appURL *url.URL,
disableGrafanaFolder bool,
maxAttempts int64,
retryConfig RetryConfig,
sender AlertsSender,
stateManager *state.Manager,
evalFactory eval.EvaluatorFactory,
@@ -75,7 +75,7 @@ func newRuleFactory(
return newRecordingRule(
ctx,
rule.GetKeyWithGroup(),
maxAttempts,
retryConfig,
clock,
evalFactory,
rrCfg,
@@ -92,7 +92,7 @@ func newRuleFactory(
rule.GetKeyWithGroup(),
appURL,
disableGrafanaFolder,
maxAttempts,
retryConfig,
sender,
stateManager,
evalFactory,
@@ -120,7 +120,7 @@ type alertRule struct {
appURL *url.URL
disableGrafanaFolder bool
maxAttempts int64
retryConfig RetryConfig
clock clock.Clock
sender AlertsSender
@@ -142,7 +142,7 @@ func newAlertRule(
key ngmodels.AlertRuleKeyWithGroup,
appURL *url.URL,
disableGrafanaFolder bool,
maxAttempts int64,
retryConfig RetryConfig,
sender AlertsSender,
stateManager *state.Manager,
evalFactory eval.EvaluatorFactory,
@@ -155,6 +155,7 @@ func newAlertRule(
stopAppliedHook func(ngmodels.AlertRuleKey),
) *alertRule {
ctx, stop := util.WithCancelCause(ngmodels.WithRuleKey(parent, key.AlertRuleKey))
return &alertRule{
key: key,
evalCh: make(chan *Evaluation),
@@ -163,7 +164,7 @@ func newAlertRule(
stopFn: stop,
appURL: appURL,
disableGrafanaFolder: disableGrafanaFolder,
maxAttempts: maxAttempts,
retryConfig: retryConfig,
clock: clock,
sender: sender,
stateManager: stateManager,
@@ -271,6 +272,14 @@ func (a *alertRule) Run() error {
logger := a.logger.New("version", ctx.rule.Version, "fingerprint", f, "now", ctx.scheduledAt)
logger.Debug("Processing tick")
retryer := newExponentialBackoffRetryer(
a.retryConfig.MaxAttempts-1, // First attempt is not a retry.
a.retryConfig.InitialRetryDelay,
a.retryConfig.MaxRetryDelay,
a.retryConfig.RandomizationFactor,
a.clock,
)
func() {
orgID := fmt.Sprint(a.key.OrgID)
evalDuration := a.metrics.EvalDuration.WithLabelValues(orgID)
@@ -282,7 +291,8 @@ func (a *alertRule) Run() error {
a.evalApplied(ctx.scheduledAt)
}()
for attempt := int64(1); attempt <= a.maxAttempts; attempt++ {
attempt := 1
for {
isPaused := ctx.rule.IsPaused
// Do not clean up state if the eval loop has just started.
@@ -327,8 +337,9 @@ func (a *alertRule) Run() error {
logger.Error("Skip evaluation and updating the state because the context has been cancelled", "version", ctx.rule.Version, "fingerprint", f, "attempt", attempt, "now", ctx.scheduledAt)
return
}
retry := attempt < a.maxAttempts
err := a.evaluate(tracingCtx, ctx, span, retry, logger)
nextDelay := retryer.NextAttemptIn()
shouldRetry := nextDelay != retryStop
err := a.evaluate(tracingCtx, ctx, span, shouldRetry, logger)
// This is extremely confusing - when we exhaust all retry attempts, or we have no retryable errors
// we return nil - so technically, this is meaningless to know whether the evaluation has errors or not.
span.End()
@@ -337,12 +348,14 @@ func (a *alertRule) Run() error {
return
}
logger.Error("Failed to evaluate rule", "attempt", attempt, "error", err)
logger.Error("Failed to evaluate rule", "attempt", attempt, "max_attempts", a.retryConfig.MaxAttempts, "next_attempt_in", nextDelay, "error", err)
attempt++
select {
case <-tracingCtx.Done():
logger.Error("Context has been cancelled while backing off", "attempt", attempt)
return
case <-time.After(retryDelay):
case <-a.clock.After(nextDelay):
continue
}
}
pkg/services/ngalert/schedule/alert_rule_test.go
+190 -19
View File
@@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/benbjohnson/clock"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
@@ -502,7 +503,7 @@ func blankRuleForTests(ctx context.Context, key models.AlertRuleKeyWithGroup) *a
Log: log.NewNopLogger(),
}
st := state.NewManager(managerCfg, state.NewNoopPersister())
return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, log.NewNopLogger(), nil, featuremgmt.WithFeatures(), nil, nil)
return newAlertRule(ctx, key, nil, false, RetryConfig{}, nil, st, nil, nil, nil, log.NewNopLogger(), nil, featuremgmt.WithFeatures(), nil, nil)
}
func TestRuleRoutine(t *testing.T) {
@@ -510,12 +511,13 @@ func TestRuleRoutine(t *testing.T) {
createSchedule := func(
evalAppliedChan chan time.Time,
senderMock *SyncAlertsSenderMock,
clk clock.Clock,
) (*schedule, *fakeRulesStore, *state.FakeInstanceStore, prometheus.Gatherer) {
ruleStore := newFakeRulesStore()
instanceStore := &state.FakeInstanceStore{}
registry := prometheus.NewPedanticRegistry()
sch := setupScheduler(t, ruleStore, instanceStore, registry, senderMock, nil, nil)
sch := setupScheduler(t, ruleStore, instanceStore, registry, senderMock, nil, nil, withSchedulerClock(clk))
sch.evalAppliedFunc = func(key models.AlertRuleKey, t time.Time) {
evalAppliedChan <- t
}
@@ -530,7 +532,7 @@ func TestRuleRoutine(t *testing.T) {
// TODO rewrite when we are able to mock/fake state manager
t.Run(fmt.Sprintf("when rule evaluation happens (evaluation state %s)", evalState), func(t *testing.T) {
evalAppliedChan := make(chan time.Time)
sch, ruleStore, instanceStore, reg := createSchedule(evalAppliedChan, nil)
sch, ruleStore, instanceStore, reg := createSchedule(evalAppliedChan, nil, clock.NewMock())
rule := gen.With(withQueryForState(t, evalState)).GenerateRef()
ruleStore.PutRule(context.Background(), rule)
@@ -718,7 +720,7 @@ func TestRuleRoutine(t *testing.T) {
t.Run("and clean up the state if parent context is cancelled", func(t *testing.T) {
stoppedChan := make(chan error)
sender := NewSyncAlertsSenderMock()
sch, _, _, _ := createSchedule(make(chan time.Time), sender)
sch, _, _, _ := createSchedule(make(chan time.Time), sender, clock.NewMock())
_ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil)
expectedStates := sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)
@@ -742,7 +744,7 @@ func TestRuleRoutine(t *testing.T) {
t.Run("and clean up the state but not send anything if the reason is not rule deleted", func(t *testing.T) {
stoppedChan := make(chan error)
sender := NewSyncAlertsSenderMock()
sch, _, _, _ := createSchedule(make(chan time.Time), sender)
sch, _, _, _ := createSchedule(make(chan time.Time), sender, clock.NewMock())
_ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil)
require.NotEmpty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID))
@@ -766,7 +768,7 @@ func TestRuleRoutine(t *testing.T) {
stoppedChan := make(chan error)
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Times(1)
sch, _, _, _ := createSchedule(make(chan time.Time), sender)
sch, _, _, _ := createSchedule(make(chan time.Time), sender, clock.NewMock())
_ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil)
require.NotEmpty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID))
@@ -796,7 +798,7 @@ func TestRuleRoutine(t *testing.T) {
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender)
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender, clock.NewMock())
ruleStore.PutRule(context.Background(), rule)
sch.schedulableAlertRules.set([]*models.AlertRule{rule}, map[models.FolderKey]string{rule.GetFolderKey(): folderTitle})
factory := ruleFactoryFromScheduler(sch)
@@ -878,10 +880,34 @@ func TestRuleRoutine(t *testing.T) {
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, reg := createSchedule(evalAppliedChan, sender)
sch.maxAttempts = 3
clk := clock.NewMock()
sch, ruleStore, _, reg := createSchedule(evalAppliedChan, sender, clk)
sch.retryConfig = RetryConfig{
MaxAttempts: 3,
InitialRetryDelay: 1 * time.Second,
MaxRetryDelay: 1 * time.Second,
RandomizationFactor: 0,
}
ruleStore.PutRule(context.Background(), rule)
factory := ruleFactoryFromScheduler(sch)
factory := newRuleFactory(
sch.appURL,
sch.disableGrafanaFolder,
sch.retryConfig,
sch.alertsSender,
sch.stateManager,
sch.evaluatorFactory,
sch.clock,
sch.rrCfg,
sch.metrics,
sch.log,
sch.tracer,
sch.featureToggles,
sch.recordingWriter,
sch.evalAppliedFunc,
sch.stopAppliedFunc,
)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
ruleInfo := factory.new(ctx, rule)
@@ -895,6 +921,12 @@ func TestRuleRoutine(t *testing.T) {
rule: rule,
})
// Because we are using a mock clock, first we need to wait until the rule evaluation
// reaches the point where it sleeps for the duration of the retry interval.
time.Sleep(200 * time.Millisecond)
// Then advance the mock clock to trigger the retry.
clk.Add(2 * time.Second)
waitForTimeChannel(t, evalAppliedChan)
t.Run("it should increase failure counter by 1 and attempt failure counter by 3", func(t *testing.T) {
@@ -902,10 +934,10 @@ func TestRuleRoutine(t *testing.T) {
expectedMetric := fmt.Sprintf(
`# HELP grafana_alerting_rule_evaluation_duration_seconds The time to evaluate a rule.
# TYPE grafana_alerting_rule_evaluation_duration_seconds histogram
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.01"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.1"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.5"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="1"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.01"} 0
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.1"} 0
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="0.5"} 0
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="1"} 0
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="5"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="10"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="15"} 1
@@ -916,7 +948,7 @@ func TestRuleRoutine(t *testing.T) {
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="240"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="300"} 1
grafana_alerting_rule_evaluation_duration_seconds_bucket{org="%[1]d",le="+Inf"} 1
grafana_alerting_rule_evaluation_duration_seconds_sum{org="%[1]d"} 0
grafana_alerting_rule_evaluation_duration_seconds_sum{org="%[1]d"} 2
grafana_alerting_rule_evaluation_duration_seconds_count{org="%[1]d"} 1
# HELP grafana_alerting_rule_evaluation_failures_total The total number of rule evaluation failures.
# TYPE grafana_alerting_rule_evaluation_failures_total counter
@@ -1007,7 +1039,7 @@ func TestRuleRoutine(t *testing.T) {
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender)
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender, clock.NewMock())
ruleStore.PutRule(context.Background(), rule)
factory := ruleFactoryFromScheduler(sch)
ctx, cancel := context.WithCancel(context.Background())
@@ -1041,7 +1073,7 @@ func TestRuleRoutine(t *testing.T) {
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender)
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender, clock.NewMock())
ruleStore.PutRule(context.Background(), rule)
factory := ruleFactoryFromScheduler(sch)
ctx, cancel := context.WithCancel(context.Background())
@@ -1076,7 +1108,7 @@ func TestRuleRoutine(t *testing.T) {
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender)
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, sender, clock.NewMock())
sch.stateManager.ResolvedRetention = 4 * time.Second
sch.stateManager.ResendDelay = 2 * time.Second
sch.stateManager.Put([]*state.State{
@@ -1125,8 +1157,147 @@ func TestRuleRoutine(t *testing.T) {
})
}
func TestAlertRuleRetry(t *testing.T) {
gen := models.RuleGen
createSchedule := func(
evalAppliedChan chan time.Time,
senderMock *SyncAlertsSenderMock,
) (*schedule, *fakeRulesStore, *state.FakeInstanceStore, prometheus.Gatherer) {
ruleStore := newFakeRulesStore()
instanceStore := &state.FakeInstanceStore{}
registry := prometheus.NewPedanticRegistry()
sch := setupScheduler(t, ruleStore, instanceStore, registry, senderMock, nil, nil)
sch.evalAppliedFunc = func(key models.AlertRuleKey, t time.Time) {
evalAppliedChan <- t
}
return sch, ruleStore, instanceStore, registry
}
evalAppliedChan := make(chan time.Time)
rule := gen.With(withQueryForState(t, eval.Error)).GenerateRef()
rule.ExecErrState = models.ErrorErrState
sender := NewSyncAlertsSenderMock()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, reg := createSchedule(evalAppliedChan, sender)
fakeClock := sch.clock.(*clock.Mock)
ruleStore.PutRule(context.Background(), rule)
maxAttempts := int64(3)
backoffDuration := time.Millisecond * 10
factory := newRuleFactory(
sch.appURL,
sch.disableGrafanaFolder,
RetryConfig{
MaxAttempts: maxAttempts,
InitialRetryDelay: backoffDuration,
MaxRetryDelay: backoffDuration,
},
sch.alertsSender,
sch.stateManager,
sch.evaluatorFactory,
fakeClock,
sch.rrCfg,
sch.metrics,
sch.log,
sch.tracer,
sch.featureToggles,
sch.recordingWriter,
sch.evalAppliedFunc,
sch.stopAppliedFunc,
)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
ruleInfo := factory.new(ctx, rule)
go func() {
_ = ruleInfo.Run()
}()
// Run the rule evaluation tick
ruleInfo.Eval(&Evaluation{
scheduledAt: sch.clock.Now(),
rule: rule,
})
compareMetrics := func(c *assert.CollectT, evaluations, expectedFailures int) {
expectedMetric := fmt.Sprintf(
`# HELP grafana_alerting_rule_evaluation_attempts_total The total number of rule evaluation attempts.
# TYPE grafana_alerting_rule_evaluation_attempts_total counter
grafana_alerting_rule_evaluation_attempts_total{org="%[1]d"} %[3]d
# HELP grafana_alerting_rule_evaluation_attempt_failures_total The total number of rule evaluation attempt failures.
# TYPE grafana_alerting_rule_evaluation_attempt_failures_total counter
grafana_alerting_rule_evaluation_attempt_failures_total{org="%[1]d"} %[3]d
# HELP grafana_alerting_rule_evaluations_total The total number of rule evaluations.
# TYPE grafana_alerting_rule_evaluations_total counter
grafana_alerting_rule_evaluations_total{org="%[1]d"} %[2]d
`, rule.OrgID, evaluations, expectedFailures)
err := testutil.GatherAndCompare(
reg,
bytes.NewBufferString(expectedMetric),
"grafana_alerting_rule_evaluations_total",
"grafana_alerting_rule_evaluation_attempts_total",
"grafana_alerting_rule_evaluation_attempt_failures_total",
)
assert.NoError(c, err)
}
t.Run("first attempt", func(t *testing.T) {
require.EventuallyWithT(t, func(c *assert.CollectT) {
compareMetrics(c, 1, 1)
}, 5*time.Millisecond, 1*time.Millisecond)
})
t.Run("second attempt", func(t *testing.T) {
// advance the clock by the backoff duration
fakeClock.Add(backoffDuration)
require.EventuallyWithT(t, func(c *assert.CollectT) {
compareMetrics(c, 1, 2)
}, 5*time.Millisecond, 1*time.Millisecond)
})
t.Run("third attempt", func(t *testing.T) {
// advance the clock by the backoff duration
fakeClock.Add(backoffDuration)
require.EventuallyWithT(t, func(c *assert.CollectT) {
compareMetrics(c, 1, 3)
}, 5*time.Millisecond, 1*time.Millisecond)
})
t.Run("no fourth attempt", func(t *testing.T) {
// Wait long enough to ensure no fourth attempt occurs
fakeClock.Add(backoffDuration * 10)
require.EventuallyWithT(t, func(c *assert.CollectT) {
compareMetrics(c, 1, 3)
}, 5*time.Millisecond, 1*time.Millisecond)
})
}
func ruleFactoryFromScheduler(sch *schedule) ruleFactory {
return newRuleFactory(sch.appURL, sch.disableGrafanaFolder, sch.maxAttempts, sch.alertsSender, sch.stateManager, sch.evaluatorFactory, sch.clock, sch.rrCfg, sch.metrics, sch.log, sch.tracer, sch.featureToggles, sch.recordingWriter, sch.evalAppliedFunc, sch.stopAppliedFunc)
return newRuleFactory(
sch.appURL,
sch.disableGrafanaFolder,
sch.retryConfig,
sch.alertsSender,
sch.stateManager,
sch.evaluatorFactory,
sch.clock,
sch.rrCfg,
sch.metrics,
sch.log,
sch.tracer,
sch.featureToggles,
sch.recordingWriter,
sch.evalAppliedFunc,
sch.stopAppliedFunc,
)
}
func stateForRule(rule *models.AlertRule, ts time.Time, evalState eval.State) *state.State {
pkg/services/ngalert/schedule/recording_rule.go
+40 -15
View File
@@ -40,7 +40,7 @@ type recordingRule struct {
evaluationTimestamp *atomic.Time
evaluationDuration *atomic.Duration
maxAttempts int64
retryConfig RetryConfig
clock clock.Clock
evalFactory eval.EvaluatorFactory
@@ -56,7 +56,20 @@ type recordingRule struct {
tracer tracing.Tracer
}
func newRecordingRule(parent context.Context, key ngmodels.AlertRuleKeyWithGroup, maxAttempts int64, clock clock.Clock, evalFactory eval.EvaluatorFactory, cfg setting.RecordingRuleSettings, logger log.Logger, metrics *metrics.Scheduler, tracer tracing.Tracer, writer RecordingWriter, evalAppliedHook evalAppliedFunc, stopAppliedHook stopAppliedFunc) *recordingRule {
func newRecordingRule(
parent context.Context,
key ngmodels.AlertRuleKeyWithGroup,
retryConfig RetryConfig,
clock clock.Clock,
evalFactory eval.EvaluatorFactory,
cfg setting.RecordingRuleSettings,
logger log.Logger,
metrics *metrics.Scheduler,
tracer tracing.Tracer,
writer RecordingWriter,
evalAppliedHook evalAppliedFunc,
stopAppliedHook stopAppliedFunc,
) *recordingRule {
ctx, stop := util.WithCancelCause(ngmodels.WithRuleKey(parent, key.AlertRuleKey))
return &recordingRule{
key: key,
@@ -70,7 +83,7 @@ func newRecordingRule(parent context.Context, key ngmodels.AlertRuleKeyWithGroup
clock: clock,
evalFactory: evalFactory,
cfg: cfg,
maxAttempts: maxAttempts,
retryConfig: retryConfig,
evalAppliedHook: evalAppliedHook,
stopAppliedHook: stopAppliedHook,
logger: logger.FromContext(ctx),
@@ -190,8 +203,17 @@ func (r *recordingRule) doEvaluate(ctx context.Context, ev *Evaluation) {
))
defer span.End()
retryer := newExponentialBackoffRetryer(
r.retryConfig.MaxAttempts-1, // first attempt is not a retry
r.retryConfig.InitialRetryDelay,
r.retryConfig.MaxRetryDelay,
r.retryConfig.RandomizationFactor,
r.clock,
)
attempt := 1
var latestError error
for attempt := int64(1); attempt <= r.maxAttempts; attempt++ {
for {
logger := logger.New("attempt", attempt)
if ctx.Err() != nil {
span.SetStatus(codes.Error, "rule evaluation cancelled")
@@ -213,14 +235,20 @@ func (r *recordingRule) doEvaluate(ctx context.Context, ev *Evaluation) {
break
}
if attempt < r.maxAttempts {
select {
case <-ctx.Done():
logger.Error("Context has been cancelled while backing off", "attempt", attempt)
return
case <-time.After(retryDelay):
continue
}
retryIn := retryer.NextAttemptIn()
if retryIn == retryStop {
logger.Error("Recording rule evaluation failed after all attempts", "lastError", latestError)
break
}
attempt++
select {
case <-ctx.Done():
logger.Error("Context has been cancelled while backing off", "attempt", attempt)
return
case <-r.clock.After(retryIn):
continue
}
}
@@ -230,9 +258,6 @@ func (r *recordingRule) doEvaluate(ctx context.Context, ev *Evaluation) {
span.RecordError(latestError)
r.lastError.Store(latestError)
r.health.Store("error")
if r.maxAttempts > 0 {
logger.Error("Recording rule evaluation failed after all attempts", "lastError", latestError)
}
return
}
logger.Debug("Recording rule evaluation succeeded")
pkg/services/ngalert/schedule/recording_rule_test.go
+18 -2
View File
@@ -178,7 +178,8 @@ func blankRecordingRuleForTests(ctx context.Context) *recordingRule {
st := setting.RecordingRuleSettings{
Enabled: true,
}
return newRecordingRule(context.Background(), models.AlertRuleKeyWithGroup{}, 0, nil, nil, st, log.NewNopLogger(), nil, nil, writer.FakeWriter{}, nil, nil)
return newRecordingRule(context.Background(), models.AlertRuleKeyWithGroup{}, RetryConfig{}, nil, nil, st, log.NewNopLogger(), nil, nil, writer.FakeWriter{}, nil, nil)
}
func TestRecordingRule_Integration(t *testing.T) {
@@ -431,7 +432,8 @@ func testRecordingRule_Integration(t *testing.T, writeTarget *writer.TestRemoteW
gen := models.RuleGen.With(models.RuleGen.WithAllRecordingRules(), models.RuleGen.WithOrgID(123))
ruleStore := newFakeRulesStore()
reg := prometheus.NewPedanticRegistry()
sch := setupScheduler(t, ruleStore, nil, reg, nil, nil, nil)
clk := clock.NewMock()
sch := setupScheduler(t, ruleStore, nil, reg, nil, nil, nil, withSchedulerClock(clk))
sch.recordingWriter = writer
t.Run("rule that succeeds", func(t *testing.T) {
@@ -606,6 +608,13 @@ func testRecordingRule_Integration(t *testing.T, writeTarget *writer.TestRemoteW
rule: rule,
folderTitle: folderTitle,
})
// Because we are using a mock clock, first we need to wait until the rule evaluation
// reaches the point where it sleeps for the duration of the retry interval.
time.Sleep(200 * time.Millisecond)
// Then advance the mock clock to trigger the retry.
clk.Add(2 * time.Second)
_ = waitForTimeChannel(t, evalDoneChan)
t.Run("reports basic evaluation metrics", func(t *testing.T) {
@@ -744,6 +753,13 @@ func testRecordingRule_Integration(t *testing.T, writeTarget *writer.TestRemoteW
rule: rule,
folderTitle: folderTitle,
})
// Because we are using a mock clock, first we need to wait until the rule evaluation
// reaches the point where it sleeps for the duration of the retry interval.
time.Sleep(200 * time.Millisecond)
// Then advance the mock clock to trigger the retry.
clk.Add(2 * time.Second)
_ = waitForTimeChannel(t, evalDoneChan)
t.Run("status shows evaluation", func(t *testing.T) {
pkg/services/ngalert/schedule/retry.go
+37
View File
@@ -0,0 +1,37 @@
package schedule
import (
"time"
"github.com/benbjohnson/clock"
"github.com/cenkalti/backoff/v4"
)
const retryStop = backoff.Stop
type exponentialBackoffRetryer struct {
backoff.BackOff
}
func newExponentialBackoffRetryer(
maxRetries int64,
initialRetryDelay time.Duration,
maxRetryDelay time.Duration,
randomizationFactor float64,
clock clock.Clock,
) *exponentialBackoffRetryer {
b := backoff.NewExponentialBackOff(
backoff.WithClockProvider(clock),
backoff.WithMaxInterval(maxRetryDelay),
backoff.WithInitialInterval(initialRetryDelay),
backoff.WithRandomizationFactor(randomizationFactor),
)
return &exponentialBackoffRetryer{
BackOff: backoff.WithMaxRetries(b, uint64(maxRetries)),
}
}
func (b *exponentialBackoffRetryer) NextAttemptIn() time.Duration {
return b.NextBackOff()
}
pkg/services/ngalert/schedule/retry_test.go
+73
View File
@@ -0,0 +1,73 @@
package schedule
import (
"testing"
"time"
"github.com/benbjohnson/clock"
"github.com/cenkalti/backoff/v4"
"github.com/stretchr/testify/require"
)
func TestExponentialBackoffRetryProvider_New(t *testing.T) {
testClock := clock.NewMock()
maxRetries := int64(5)
initialDelay := 100 * time.Millisecond
maxDelay := 1 * time.Second
retry := newExponentialBackoffRetryer(maxRetries, initialDelay, maxDelay, 0, testClock)
require.NotNil(t, retry, "Retry instance should not be nil")
for i := int64(0); i < maxRetries; i++ {
delay := retry.NextAttemptIn()
require.GreaterOrEqual(t, delay, initialDelay, "Delay should be at least the initial delay")
require.LessOrEqual(t, delay, maxDelay, "Delay should not exceed the max delay")
testClock.Add(delay)
}
delay := retry.NextAttemptIn()
require.Equal(t, backoff.Stop, delay, "Delay should be backoff.Stop after max retries")
}
func TestExponentialBackoffRetryProvider_MaxRetries(t *testing.T) {
testClock := clock.NewMock()
t.Run("max retries is zero", func(t *testing.T) {
retry := newExponentialBackoffRetryer(0, 100*time.Millisecond, 1*time.Second, 0, testClock)
require.NotNil(t, retry, "Retry instance should not be nil")
delay := retry.NextAttemptIn()
require.Equal(t, backoff.Stop, delay, "Should immediately stop when maxRetries is 0")
})
t.Run("max retries is not zero", func(t *testing.T) {
maxRetries := int64(10)
retry := newExponentialBackoffRetryer(maxRetries, 10*time.Millisecond, 1*time.Second, 0, testClock)
for i := int64(0); i < maxRetries; i++ {
delay := retry.NextAttemptIn()
require.NotEqual(t, backoff.Stop, delay, "Should not stop before reaching max retries")
testClock.Add(delay)
}
delay := retry.NextAttemptIn()
require.Equal(t, backoff.Stop, delay, "Should stop after reaching maxRetries")
})
}
func TestExponentialBackoffRetryProvider_DelaysWithinBounds(t *testing.T) {
testClock := clock.NewMock()
initialDelay := 200 * time.Millisecond
maxDelay := 2 * time.Second
maxRetries := int64(10)
retry := newExponentialBackoffRetryer(maxRetries, initialDelay, maxDelay, 0, testClock)
for i := int64(0); i < maxRetries; i++ {
delay := retry.NextAttemptIn()
require.GreaterOrEqual(t, delay, initialDelay, "Delay should not be less than initial delay")
require.LessOrEqual(t, delay, maxDelay, "Delay should not exceed max delay")
testClock.Add(delay)
}
}
pkg/services/ngalert/schedule/schedule.go
+17 -11
View File
@@ -31,9 +31,6 @@ type ScheduleService interface {
Run(context.Context) error
}
// retryDelay represents how long to wait between each failed rule evaluation.
const retryDelay = 1 * time.Second
// AlertsSender is an interface for a service that is responsible for sending notifications to the end-user.
//
//go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter
@@ -67,7 +64,7 @@ type schedule struct {
// each rule gets its own channel and routine
registry ruleRegistry
maxAttempts int64
retryConfig RetryConfig
clock clock.Clock
@@ -112,9 +109,17 @@ type schedule struct {
recordingWriter RecordingWriter
}
// RetryConfig configures the exponential backoff for alert rule and recording rule evaluations.
type RetryConfig struct {
MaxAttempts int64
InitialRetryDelay time.Duration
MaxRetryDelay time.Duration
RandomizationFactor float64
}
// SchedulerCfg is the scheduler configuration.
type SchedulerCfg struct {
MaxAttempts int64
RetryConfig RetryConfig
BaseInterval time.Duration
C clock.Clock
MinRuleInterval time.Duration
@@ -136,14 +141,14 @@ type SchedulerCfg struct {
// NewScheduler returns a new scheduler.
func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule {
const minMaxAttempts = int64(1)
if cfg.MaxAttempts < minMaxAttempts {
cfg.Log.Warn("Invalid scheduler maxAttempts, using a safe minimum", "configured", cfg.MaxAttempts, "actual", minMaxAttempts)
cfg.MaxAttempts = minMaxAttempts
if cfg.RetryConfig.MaxAttempts < minMaxAttempts {
cfg.Log.Warn("Invalid scheduler maxAttempts, using a safe minimum", "configured", cfg.RetryConfig.MaxAttempts, "actual", minMaxAttempts)
cfg.RetryConfig.MaxAttempts = minMaxAttempts
}
sch := schedule{
registry: newRuleRegistry(),
maxAttempts: cfg.MaxAttempts,
retryConfig: cfg.RetryConfig,
clock: cfg.C,
baseInterval: cfg.BaseInterval,
log: cfg.Log,
@@ -168,7 +173,7 @@ func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule {
}
func (sch *schedule) Run(ctx context.Context) error {
sch.log.Info("Starting scheduler", "tickInterval", sch.baseInterval, "maxAttempts", sch.maxAttempts)
sch.log.Info("Starting scheduler", "tickInterval", sch.baseInterval, "maxAttempts", sch.retryConfig.MaxAttempts)
t := ticker.New(sch.clock, sch.baseInterval, sch.metrics.Ticker, sch.log)
defer t.Stop()
@@ -296,10 +301,11 @@ func (sch *schedule) processTick(ctx context.Context, dispatcherGroup *errgroup.
updatedRules := make([]ngmodels.AlertRuleKeyWithVersion, 0, len(updated)) // this is needed for tests only
restartedRules := make([]Rule, 0)
missingFolder := make(map[string][]string)
ruleFactory := newRuleFactory(
sch.appURL,
sch.disableGrafanaFolder,
sch.maxAttempts,
sch.retryConfig,
sch.alertsSender,
sch.stateManager,
sch.evaluatorFactory,
pkg/services/ngalert/schedule/schedule_unit_test.go
+45 -7
View File
@@ -85,6 +85,12 @@ func TestProcessTicks(t *testing.T) {
}
schedCfg := SchedulerCfg{
RetryConfig: RetryConfig{
MaxAttempts: 1,
InitialRetryDelay: time.Second,
MaxRetryDelay: time.Second * 10,
RandomizationFactor: 0,
},
BaseInterval: cfg.BaseInterval,
C: mockedClock,
AppURL: appUrl,
@@ -1189,11 +1195,35 @@ func TestSchedule_deleteAlertRule(t *testing.T) {
})
}
func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStore, registry *prometheus.Registry, senderMock *SyncAlertsSenderMock, evalMock eval.EvaluatorFactory, ruleStopReasonProvider AlertRuleStopReasonProvider) *schedule {
type schedulerOpts struct {
clock clock.Clock
}
func withSchedulerClock(clock clock.Clock) func(opts *schedulerOpts) {
return func(opts *schedulerOpts) {
opts.clock = clock
}
}
func setupScheduler(
t *testing.T,
rs *fakeRulesStore,
is *state.FakeInstanceStore,
registry *prometheus.Registry,
senderMock *SyncAlertsSenderMock,
evalMock eval.EvaluatorFactory,
ruleStopReasonProvider AlertRuleStopReasonProvider,
options ...func(opts *schedulerOpts),
) *schedule {
t.Helper()
testTracer := tracing.InitializeTracerForTest()
mockedClock := clock.NewMock()
opts := &schedulerOpts{
clock: clock.NewMock(),
}
for _, o := range options {
o(opts)
}
if rs == nil {
rs = newFakeRulesStore()
@@ -1236,8 +1266,11 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
}
cfg := setting.UnifiedAlertingSettings{
BaseInterval: time.Second,
MaxAttempts: 1,
BaseInterval: time.Second,
MaxAttempts: 1,
InitialRetryDelay: time.Second * 1,
MaxRetryDelay: time.Second * 10,
RandomizationFactor: 0,
RecordingRules: setting.RecordingRuleSettings{
Enabled: true,
},
@@ -1246,9 +1279,14 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
fakeRecordingWriter := writer.FakeWriter{}
schedCfg := SchedulerCfg{
RetryConfig: RetryConfig{
MaxAttempts: cfg.MaxAttempts,
InitialRetryDelay: cfg.InitialRetryDelay,
MaxRetryDelay: cfg.MaxRetryDelay,
RandomizationFactor: cfg.RandomizationFactor,
},
BaseInterval: cfg.BaseInterval,
MaxAttempts: cfg.MaxAttempts,
C: mockedClock,
C: opts.clock,
AppURL: appUrl,
EvaluatorFactory: evaluator,
RuleStore: rs,
@@ -1266,7 +1304,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
ExternalURL: nil,
InstanceStore: is,
Images: &state.NoopImageService{},
Clock: mockedClock,
Clock: opts.clock,
Historian: &state.FakeHistorian{},
Tracer: testTracer,
Log: log.New("ngalert.state.manager"),
pkg/setting/setting_unified_alerting.go
+27
View File
@@ -51,6 +51,9 @@ const (
schedulerDefaultAdminConfigPollInterval = time.Minute
schedulerDefaultExecuteAlerts = true
schedulerDefaultMaxAttempts = 3
schedulerDefaultInitialRetryDelay = 1 * time.Second
schedulerDefaultMaxRetryDelay = 4 * time.Second
schedulerDefaultRandomizationFactor = 0.1
schedulerDefaultLegacyMinInterval = 1
screenshotsDefaultCapture = false
screenshotsDefaultCaptureTimeout = 10 * time.Second
@@ -106,6 +109,9 @@ type UnifiedAlertingSettings struct {
HARedisTLSConfig dstls.ClientConfig
InitializationTimeout time.Duration
MaxAttempts int64
InitialRetryDelay time.Duration
MaxRetryDelay time.Duration
RandomizationFactor float64
MinInterval time.Duration
EvaluationTimeout time.Duration
EvaluationResultLimit int
@@ -363,6 +369,27 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfg.MaxAttempts = ua.Key("max_attempts").MustInt64(schedulerDefaultMaxAttempts)
uaInitialRetryDelay, err := gtime.ParseDuration(valueAsString(ua, "initial_retry_delay", schedulerDefaultInitialRetryDelay.String()))
if err != nil {
cfg.Logger.Warn("failed to parse setting 'initial_retry_delay' as duration, falling back to the default value", "error", err, "default", schedulerDefaultInitialRetryDelay)
uaInitialRetryDelay = schedulerDefaultInitialRetryDelay
}
uaCfg.InitialRetryDelay = uaInitialRetryDelay
uaMaxRetryDelay, err := gtime.ParseDuration(valueAsString(ua, "max_retry_delay", schedulerDefaultMaxRetryDelay.String()))
if err != nil {
cfg.Logger.Warn("failed to parse setting 'max_retry_delay' as duration, falling back to the default value", "error", err, "default", schedulerDefaultMaxRetryDelay)
uaMaxRetryDelay = schedulerDefaultMaxRetryDelay
}
uaCfg.MaxRetryDelay = uaMaxRetryDelay
uaRandomizationFactor := ua.Key("randomization_factor").MustFloat64(schedulerDefaultRandomizationFactor)
if uaRandomizationFactor < 0 || uaRandomizationFactor > 1 {
cfg.Logger.Warn("randomization_factor must be between 0 and 1, falling back to the default value", "value", uaRandomizationFactor, "default", schedulerDefaultRandomizationFactor)
uaRandomizationFactor = schedulerDefaultRandomizationFactor
}
uaCfg.RandomizationFactor = uaRandomizationFactor
uaCfg.BaseInterval = SchedulerBaseInterval
// TODO: This was promoted from a feature toggle and is now the default behavior.