Alerting: new min_interval_seconds options to enforce a minimum eval frequency (#21188)

* add min_interval_seconds setting to alerting config

It will let operator enforce a minimum time for the scheduler to enqueue evaluations

* Introduce UI modifications

* Update docs

Co-authored-by: Martin <uepoch@users.noreply.github.com>

pkg/api/frontendsettings.go

@@ -176,6 +176,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *m.ReqContext) (map[string]interf
 		"alertingEnabled":            setting.AlertingEnabled,
 		"alertingErrorOrTimeout":     setting.AlertingErrorOrTimeout,
 		"alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues,
+		"alertingMinInterval":        setting.AlertingMinInterval,
 		"exploreEnabled":             setting.ExploreEnabled,
 		"googleAnalyticsId":          setting.GoogleAnalyticsId,
 		"disableLoginForm":           setting.DisableLoginForm,

pkg/services/alerting/scheduler.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 type schedulerImpl struct {
@@ -61,7 +62,13 @@ func (s *schedulerImpl) Tick(tickTime time.Time, execQueue chan *Job) {
 			continue
 		}
 
-		if now%job.Rule.Frequency == 0 {
+		// Check the job frequency against the minimum interval required
+		interval := job.Rule.Frequency
+		if interval < setting.AlertingMinInterval {
+			interval = setting.AlertingMinInterval
+		}
+
+		if now%interval == 0 {
 			if job.Offset > 0 {
 				job.OffsetWait = true
 			} else {

pkg/setting/setting.go

@@ -201,6 +201,7 @@ var (
 	AlertingEvaluationTimeout   time.Duration
 	AlertingNotificationTimeout time.Duration
 	AlertingMaxAttempts         int
+	AlertingMinInterval         int64
 
 	// Explore UI
 	ExploreEnabled bool
@@ -961,6 +962,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	notificationTimeoutSeconds := alerting.Key("notification_timeout_seconds").MustInt64(30)
 	AlertingNotificationTimeout = time.Second * time.Duration(notificationTimeoutSeconds)
 	AlertingMaxAttempts = alerting.Key("max_attempts").MustInt(3)
+	AlertingMinInterval = alerting.Key("min_interval_seconds").MustInt64(1)
 
 	explore := iniFile.Section("explore")
 	ExploreEnabled = explore.Key("enabled").MustBool(true)