Alerting: Add setting to distribute rule group evaluations over time (#80766)

* Simple, per-base-interval jitter

* Add log just for test purposes

* Add strategy approach, allow choosing between group or rule

* Add flag to jitter rules

* Add second toggle for jittering within a group

* Wire up toggles to strategy

* Slightly improve comment ordering

* Add tests for offset generation

* Rename JitterStrategyFrom

* Improve debug log message

* Use grafana SDK labels rather than prometheus labels

pkg/services/ngalert/schedule/jitter.go

@@ -0,0 +1,66 @@
+package schedule
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+)
+
+// JitterStrategy represents a modifier to alert rule timing that affects how evaluations are distributed.
+type JitterStrategy int
+
+const (
+	JitterNever JitterStrategy = iota
+	JitterByGroup
+	JitterByRule
+)
+
+// JitterStrategyFrom returns the JitterStrategy indicated by the current Grafana feature toggles.
+func JitterStrategyFrom(toggles featuremgmt.FeatureToggles) JitterStrategy {
+	strategy := JitterNever
+	if toggles.IsEnabledGlobally(featuremgmt.FlagJitterAlertRules) {
+		strategy = JitterByGroup
+	}
+	if toggles.IsEnabledGlobally(featuremgmt.FlagJitterAlertRulesWithinGroups) {
+		strategy = JitterByRule
+	}
+	return strategy
+}
+
+// jitterOffsetInTicks gives the jitter offset for a rule, in terms of a number of ticks relative to its interval and a base interval.
+// The resulting number of ticks is non-negative. We assume the rule is well-formed and has an IntervalSeconds greater to or equal than baseInterval.
+func jitterOffsetInTicks(r *ngmodels.AlertRule, baseInterval time.Duration, strategy JitterStrategy) int64 {
+	if strategy == JitterNever {
+		return 0
+	}
+
+	itemFrequency := r.IntervalSeconds / int64(baseInterval.Seconds())
+	offset := jitterHash(r, strategy) % uint64(itemFrequency)
+	// Offset is always nonnegative and less than int64.max, because above we mod by itemFrequency which fits in the positive half of int64.
+	// offset <= itemFrequency <= int64.max
+	// So, this will not overflow and produce a negative offset.
+	res := int64(offset)
+
+	// Regardless, take an absolute value anyway for an extra layer of safety in case the above logic ever changes.
+	// Our contract requires that the result is nonnegative and less than int64.max.
+	if res < 0 {
+		return -res
+	}
+	return res
+}
+
+func jitterHash(r *ngmodels.AlertRule, strategy JitterStrategy) uint64 {
+	ls := data.Labels{
+		"name":  r.RuleGroup,
+		"file":  r.NamespaceUID,
+		"orgId": fmt.Sprint(r.OrgID),
+	}
+
+	if strategy == JitterByRule {
+		ls["uid"] = r.UID
+	}
+	return uint64(ls.Fingerprint())
+}