public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Move BaseInterval and MinInterval to UnifiedAlerting config (#45107)

Browse Source 
* use base interval if legacy value is less than the base interval
This commit is contained in:
Yuriy Tseretyan
2022-02-11 16:13:49 -05:00
committed by GitHub
parent 4f815e3d8e
commit 095ea44e97
6 changed files with 145 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/setting/setting.go
-6
View File
@@ -418,12 +418,6 @@ type Cfg struct {
// Grafana.com URL
GrafanaComURL string
// Alerting
// AlertingBaseInterval controls the alerting base interval in seconds.
// Only for internal use and not user configuration.
AlertingBaseInterval time.Duration
// Geomap base layer config
GeomapDefaultBaseLayerConfig map[string]interface{}
GeomapEnableCustomBaseLayers bool
pkg/setting/setting_unified_alerting.go
+34 -6
View File
@@ -2,6 +2,7 @@ package setting
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
@@ -47,7 +48,12 @@ const (
schedulereDefaultExecuteAlerts = true
schedulerDefaultMaxAttempts = 3
schedulerDefaultLegacyMinInterval = 1
schedulerDefaultMinInterval = 10 * time.Second
// SchedulerBaseInterval base interval of the scheduler. Controls how often the scheduler fetches database for new changes as well as schedules evaluation of a rule
// changing this value is discouraged because this could cause existing alert definition
// with intervals that are not exactly divided by this number not to be evaluated
SchedulerBaseInterval = 10 * time.Second
// DefaultAlertForDuration indicates a default interval of for how long a rule should be evaluated to change state from Pending to Alerting
DefaultAlertForDuration = 60 * time.Second
)
type UnifiedAlertingSettings struct {
@@ -66,6 +72,11 @@ type UnifiedAlertingSettings struct {
DefaultConfiguration string
Enabled *bool // determines whether unified alerting is enabled. If it is nil then user did not define it and therefore its value will be determined during migration. Services should not use it directly.
DisabledOrgs map[int64]struct{}
// BaseInterval interval of time the scheduler updates the rules and evaluates rules.
// Only for internal use and not user configuration.
BaseInterval time.Duration
// DefaultAlertForDuration default time for how long an alert rule should be evaluated before change state.
DefaultAlertForDuration time.Duration
}
// IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
@@ -205,17 +216,34 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
}
uaCfg.MaxAttempts = uaMaxAttempts
uaMinInterval, err := gtime.ParseDuration(valueAsString(ua, "min_interval", schedulerDefaultMinInterval.String()))
if err != nil || uaMinInterval == schedulerDefaultMinInterval { // unified option is invalid duration or equals the default
uaCfg.BaseInterval = SchedulerBaseInterval
uaMinInterval, err := gtime.ParseDuration(valueAsString(ua, "min_interval", uaCfg.BaseInterval.String()))
if err != nil || uaMinInterval == uaCfg.BaseInterval { // unified option is invalid duration or equals the default
// if the legacy option is invalid, fallback to 10 (unified alerting min interval default)
legacyMinInterval := time.Duration(alerting.Key("min_interval_seconds").MustInt64(int64(schedulerDefaultMinInterval.Seconds()))) * time.Second
if legacyMinInterval != schedulerDefaultLegacyMinInterval {
legacyMinInterval := time.Duration(alerting.Key("min_interval_seconds").MustInt64(int64(uaCfg.BaseInterval.Seconds()))) * time.Second
if legacyMinInterval > uaCfg.BaseInterval {
cfg.Logger.Warn("falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled.")
uaMinInterval = legacyMinInterval
} else {
// if legacy interval is smaller than the base interval, adjust it to the base interval
uaMinInterval = uaCfg.BaseInterval
}
uaMinInterval = legacyMinInterval
}
if uaMinInterval < uaCfg.BaseInterval {
return fmt.Errorf("value of setting 'min_interval' should be greater than the base interval (%v)", uaCfg.BaseInterval)
}
if uaMinInterval%uaCfg.BaseInterval != 0 {
return fmt.Errorf("value of setting 'min_interval' should be times of base interval (%v)", uaCfg.BaseInterval)
}
uaCfg.MinInterval = uaMinInterval
uaCfg.DefaultAlertForDuration = DefaultAlertForDuration
if uaMinInterval > uaCfg.DefaultAlertForDuration {
uaCfg.DefaultAlertForDuration = uaMinInterval
}
cfg.UnifiedAlerting = uaCfg
return nil
}
pkg/setting/setting_unified_alerting_test.go
+104 -2
View File
@@ -1,6 +1,7 @@
package setting
import (
"math/rand"
"strconv"
"testing"
"time"
@@ -68,6 +69,8 @@ func TestUnifiedAlertingSettings(t *testing.T) {
require.Equal(t, 60*time.Second, cfg.UnifiedAlerting.MinInterval)
require.Equal(t, false, cfg.UnifiedAlerting.ExecuteAlerts)
require.Equal(t, 90*time.Second, cfg.UnifiedAlerting.EvaluationTimeout)
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.BaseInterval)
require.Equal(t, DefaultAlertForDuration, cfg.UnifiedAlerting.DefaultAlertForDuration)
},
},
{
@@ -75,7 +78,7 @@ func TestUnifiedAlertingSettings(t *testing.T) {
unifiedAlertingOptions: map[string]string{
"admin_config_poll_interval": "120s",
"max_attempts": strconv.FormatInt(schedulerDefaultMaxAttempts, 10),
"min_interval": schedulerDefaultMinInterval.String(),
"min_interval": SchedulerBaseInterval.String(),
"execute_alerts": strconv.FormatBool(schedulereDefaultExecuteAlerts),
"evaluation_timeout": evaluatorDefaultEvaluationTimeout.String(),
},
@@ -91,6 +94,8 @@ func TestUnifiedAlertingSettings(t *testing.T) {
require.Equal(t, 120*time.Second, cfg.UnifiedAlerting.MinInterval)
require.Equal(t, true, cfg.UnifiedAlerting.ExecuteAlerts)
require.Equal(t, 160*time.Second, cfg.UnifiedAlerting.EvaluationTimeout)
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.BaseInterval)
require.Equal(t, 120*time.Second, cfg.UnifiedAlerting.DefaultAlertForDuration)
},
},
{
@@ -110,9 +115,11 @@ func TestUnifiedAlertingSettings(t *testing.T) {
verifyCfg: func(t *testing.T, cfg Cfg) {
require.Equal(t, alertmanagerDefaultConfigPollInterval, cfg.UnifiedAlerting.AdminConfigPollInterval)
require.Equal(t, int64(schedulerDefaultMaxAttempts), cfg.UnifiedAlerting.MaxAttempts)
require.Equal(t, schedulerDefaultMinInterval, cfg.UnifiedAlerting.MinInterval)
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.MinInterval)
require.Equal(t, schedulereDefaultExecuteAlerts, cfg.UnifiedAlerting.ExecuteAlerts)
require.Equal(t, evaluatorDefaultEvaluationTimeout, cfg.UnifiedAlerting.EvaluationTimeout)
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.BaseInterval)
require.Equal(t, DefaultAlertForDuration, cfg.UnifiedAlerting.DefaultAlertForDuration)
},
},
{
@@ -135,6 +142,8 @@ func TestUnifiedAlertingSettings(t *testing.T) {
require.Equal(t, 120*time.Second, cfg.UnifiedAlerting.MinInterval)
require.Equal(t, false, cfg.UnifiedAlerting.ExecuteAlerts)
require.Equal(t, 160*time.Second, cfg.UnifiedAlerting.EvaluationTimeout)
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.BaseInterval)
require.Equal(t, 120*time.Second, cfg.UnifiedAlerting.DefaultAlertForDuration)
},
},
}
@@ -162,3 +171,96 @@ func TestUnifiedAlertingSettings(t *testing.T) {
})
}
}
func TestMinInterval(t *testing.T) {
randPredicate := func(predicate func(dur time.Duration) bool) *time.Duration {
for {
v := time.Duration(rand.Intn(99)+1) * time.Second
if predicate(v) {
return &v
}
}
}
testCases := []struct {
desc string
minInterval *time.Duration
legacyMinInterval *time.Duration
verifyCfg func(*testing.T, *Cfg, error)
}{
{
desc: "should fail if min interval is less than base interval",
minInterval: randPredicate(func(dur time.Duration) bool { return dur < SchedulerBaseInterval }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.Error(t, err)
require.Contains(t, err.Error(), "min_interval")
},
},
{
desc: "should fail if min interval is not multiple of base interval",
minInterval: randPredicate(func(dur time.Duration) bool { return dur > SchedulerBaseInterval && dur%SchedulerBaseInterval != 0 }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.Error(t, err)
require.Contains(t, err.Error(), "min_interval")
},
},
{
desc: "should not fail if min interval is multiple of base interval",
minInterval: randPredicate(func(dur time.Duration) bool { return dur > SchedulerBaseInterval && dur%SchedulerBaseInterval == 0 }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.NoError(t, err)
},
},
{
desc: "should fail if fallback to legacy min interval and it is not multiple of base interval",
legacyMinInterval: randPredicate(func(dur time.Duration) bool { return dur > SchedulerBaseInterval && dur%SchedulerBaseInterval != 0 }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.Error(t, err)
require.Contains(t, err.Error(), "min_interval")
},
},
{
desc: "should not fail if fallback to legacy min interval it is multiple of base interval",
legacyMinInterval: randPredicate(func(dur time.Duration) bool { return dur >= SchedulerBaseInterval && dur%SchedulerBaseInterval == 0 }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.NoError(t, err)
},
},
{
desc: "should adjust DefaultAlertForDuration to min interval if it is greater",
minInterval: randPredicate(func(dur time.Duration) bool { return dur%SchedulerBaseInterval == 0 && dur > DefaultAlertForDuration }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.Equal(t, cfg.UnifiedAlerting.MinInterval, cfg.UnifiedAlerting.DefaultAlertForDuration)
},
},
{
desc: "should fallback to the default if legacy interval is less than base",
legacyMinInterval: randPredicate(func(dur time.Duration) bool { return dur < SchedulerBaseInterval }),
verifyCfg: func(t *testing.T, cfg *Cfg, err error) {
require.Equal(t, SchedulerBaseInterval, cfg.UnifiedAlerting.MinInterval)
},
},
}
for _, testCase := range testCases {
t.Run(testCase.desc, func(t *testing.T) {
f := ini.Empty()
if testCase.minInterval != nil {
section, err := f.NewSection("unified_alerting")
require.NoError(t, err)
_, err = section.NewKey("min_interval", testCase.minInterval.String())
require.NoError(t, err)
}
if testCase.legacyMinInterval != nil {
alertingSec, err := f.NewSection("alerting")
require.NoError(t, err)
_, err = alertingSec.NewKey("min_interval_seconds", strconv.Itoa(int(testCase.legacyMinInterval.Seconds())))
require.NoError(t, err)
}
cfg := NewCfg()
cfg.IsFeatureToggleEnabled = func(key string) bool { return false }
err := cfg.ReadUnifiedAlertingSettings(f)
testCase.verifyCfg(t, cfg, err)
})
}
}