From 6ea6c611a9545eb30b6501df388fe71324c850ea Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:24:44 -0600 Subject: [PATCH] Fix incorrect metric values for scheduler_behind_seconds (#45830) (#45904) (cherry picked from commit 6cccbb5a0973f66b8e4fc0faa350994c7b48e20e) Co-authored-by: George Robinson --- pkg/services/ngalert/schedule/schedule.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index eaec8aefbb4..8e8c5fd220c 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -368,7 +368,11 @@ func (sch *schedule) schedulePeriodic(ctx context.Context) error { for { select { case tick := <-sch.ticker.C: - start := time.Now() + // We use Round(0) on the start time to remove the monotonic clock. + // This is required as late ticks from the ticker have current monotonic + // timestamps such that start.Sub(tick) does not return the expected + // delta. + start := time.Now().Round(0) sch.metrics.BehindSeconds.Set(start.Sub(tick).Seconds()) tickNum := tick.Unix() / int64(sch.baseInterval.Seconds())