diff --git a/pkg/services/alerting/ticker.go b/pkg/services/alerting/ticker.go index a992a579ef7..d0f73ebb7be 100644 --- a/pkg/services/alerting/ticker.go +++ b/pkg/services/alerting/ticker.go @@ -32,7 +32,7 @@ func NewTicker(c clock.Clock, interval time.Duration, metric *metrics.Ticker) *T t := &Ticker{ C: make(chan time.Time), clock: c, - last: c.Now(), + last: getStartTick(c, interval), interval: interval, metrics: metric, stopCh: make(chan struct{}), @@ -42,9 +42,14 @@ func NewTicker(c clock.Clock, interval time.Duration, metric *metrics.Ticker) *T return t } +func getStartTick(clk clock.Clock, interval time.Duration) time.Time { + nano := clk.Now().UnixNano() + return time.Unix(0, nano-(nano%interval.Nanoseconds())) +} + func (t *Ticker) run() { logger := log.New("ticker") - logger.Info("starting") + logger.Info("starting", "first_tick", t.last.Add(t.interval)) LOOP: for { next := t.last.Add(t.interval) // calculate the time of the next tick diff --git a/pkg/services/alerting/ticker_test.go b/pkg/services/alerting/ticker_test.go index 1994939b5ec..f759d5019ab 100644 --- a/pkg/services/alerting/ticker_test.go +++ b/pkg/services/alerting/ticker_test.go @@ -35,9 +35,24 @@ func TestTicker(t *testing.T) { } return time.Time{} } - t.Run("should not drop ticks", func(t *testing.T) { + t.Run("should align with clock", func(t *testing.T) { + interval := 10 * time.Second clk := clock.NewMock() + clk.Add(1 * time.Minute) + + require.Equal(t, clk.Now(), getStartTick(clk, interval)) + now := clk.Now() + for i := 0; i < 100; i++ { + delta := time.Duration(rand.Int63n(interval.Nanoseconds())) + clk.Set(now.Add(delta)) + require.Equal(t, now, getStartTick(clk, interval)) + } + }) + + t.Run("should not drop ticks", func(t *testing.T) { interval := time.Duration(rand.Int63n(100)+10) * time.Second + clk := clock.NewMock() + clk.Add(interval) // align clock with the start tick ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry())) ticks := rand.Intn(9) + 1 @@ -119,7 +134,7 @@ func TestTicker(t *testing.T) { interval := time.Duration(rand.Int63n(9)+1) * time.Second registry := prometheus.NewPedanticRegistry() ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(registry)) - expectedTick := clk.Now().Add(interval) + expectedTick := getStartTick(clk, interval).Add(interval) expectedMetricFmt := `# HELP grafana_alerting_ticker_interval_seconds Interval at which the ticker is meant to tick. # TYPE grafana_alerting_ticker_interval_seconds gauge diff --git a/pkg/services/ngalert/CHANGELOG.md b/pkg/services/ngalert/CHANGELOG.md index 8399d105de4..cf356cf0d65 100644 --- a/pkg/services/ngalert/CHANGELOG.md +++ b/pkg/services/ngalert/CHANGELOG.md @@ -46,6 +46,7 @@ Scopes must have an order to ensure consistency and ease of search, this helps u ## Grafana Alerting - main / unreleased - [ENHANCEMENT] Scheduler: Drop ticks if rule evaluation is too slow and adds a metric grafana_alerting_schedule_rule_evaluations_missed_total to track missed evaluations per rule #48885 +- [ENHANCEMENT] Ticker to tick at predictable time #50197 ## 9.0.0