diff --git a/conf/defaults.ini b/conf/defaults.ini index 628b02ad8dc..15adad09107 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1175,6 +1175,9 @@ enabled = # Comma-separated list of organization IDs for which to disable unified alerting. Only supported if unified alerting is enabled. disabled_orgs = +# Specify how long to wait for the alerting service to initialize +initialization_timeout = 30s + # Specify the frequency of polling for admin config changes. # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m. admin_config_poll_interval = 60s diff --git a/conf/sample.ini b/conf/sample.ini index 0aebeb84806..ff558e43b33 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -1101,6 +1101,9 @@ # Comma-separated list of organization IDs for which to disable unified alerting. Only supported if unified alerting is enabled. ;disabled_orgs = +# Specify how long to wait for the alerting service to initialize +;initialization_timeout = 30s + # Specify the frequency of polling for admin config changes. # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m. ;admin_config_poll_interval = 60s diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 5e7640bf535..555305745bc 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -151,7 +151,7 @@ type AlertNG struct { func (ng *AlertNG) init() error { // AlertNG should be initialized before the cancellation deadline of initCtx - initCtx, cancelFunc := context.WithTimeout(context.Background(), 30*time.Second) + initCtx, cancelFunc := context.WithTimeout(context.Background(), ng.Cfg.UnifiedAlerting.InitializationTimeout) defer cancelFunc() ng.store.Logger = ng.Log diff --git a/pkg/services/ngalert/tests/util.go b/pkg/services/ngalert/tests/util.go index dcfe48c0c87..4d3175af28e 100644 --- a/pkg/services/ngalert/tests/util.go +++ b/pkg/services/ngalert/tests/util.go @@ -45,7 +45,8 @@ func SetupTestEnv(tb testing.TB, baseInterval time.Duration) (*ngalert.AlertNG, cfg := setting.NewCfg() cfg.UnifiedAlerting = setting.UnifiedAlertingSettings{ - BaseInterval: setting.SchedulerBaseInterval, + BaseInterval: setting.SchedulerBaseInterval, + InitializationTimeout: 30 * time.Second, } // AlertNG database migrations run and the relative database tables are created only when it's enabled cfg.UnifiedAlerting.Enabled = new(bool) diff --git a/pkg/services/quota/quotaimpl/quota_test.go b/pkg/services/quota/quotaimpl/quota_test.go index 4b4696a5bf5..6f48484f6f2 100644 --- a/pkg/services/quota/quotaimpl/quota_test.go +++ b/pkg/services/quota/quotaimpl/quota_test.go @@ -496,6 +496,7 @@ func setupEnv(t *testing.T, sqlStore db.DB, cfg *setting.Cfg, b bus.Bus, quotaSe ac := acimpl.ProvideAccessControl(featuremgmt.WithFeatures()) ruleStore, err := ngstore.ProvideDBStore(cfg, featuremgmt.WithFeatures(), sqlStore, &foldertest.FakeService{}, &dashboards.FakeDashboardService{}, ac) require.NoError(t, err) + cfg.UnifiedAlerting.InitializationTimeout = 30 * time.Second _, err = ngalert.ProvideService( cfg, featuremgmt.WithFeatures(), nil, nil, routing.NewRouteRegister(), sqlStore, ngalertfakes.NewFakeKVStore(t), nil, nil, quotaService, secretsService, nil, m, &foldertest.FakeService{}, &acmock.Mock{}, &dashboards.FakeDashboardService{}, nil, b, &acmock.Mock{}, diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index d6f674979e3..4e78498ea30 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -43,6 +43,7 @@ const ( } } ` + alertingDefaultInitializationTimeout = 30 * time.Second evaluatorDefaultEvaluationTimeout = 30 * time.Second schedulerDefaultAdminConfigPollInterval = time.Minute schedulerDefaultExecuteAlerts = true @@ -83,6 +84,7 @@ type UnifiedAlertingSettings struct { HARedisMaxConns int HARedisTLSEnabled bool HARedisTLSConfig dstls.ClientConfig + InitializationTimeout time.Duration MaxAttempts int64 MinInterval time.Duration EvaluationTimeout time.Duration @@ -200,6 +202,11 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error { uaCfg.DisabledOrgs[orgID] = struct{}{} } + uaCfg.InitializationTimeout, err = gtime.ParseDuration(valueAsString(ua, "initialization_timeout", (alertingDefaultInitializationTimeout).String())) + if err != nil { + return err + } + uaCfg.AdminConfigPollInterval, err = gtime.ParseDuration(valueAsString(ua, "admin_config_poll_interval", (schedulerDefaultAdminConfigPollInterval).String())) if err != nil { return err diff --git a/pkg/setting/setting_unified_alerting_test.go b/pkg/setting/setting_unified_alerting_test.go index 4f4a540b401..04117305302 100644 --- a/pkg/setting/setting_unified_alerting_test.go +++ b/pkg/setting/setting_unified_alerting_test.go @@ -25,6 +25,7 @@ func TestCfg_ReadUnifiedAlertingSettings(t *testing.T) { require.Len(t, cfg.UnifiedAlerting.HAPeers, 0) require.Equal(t, 200*time.Millisecond, cfg.UnifiedAlerting.HAGossipInterval) require.Equal(t, time.Minute, cfg.UnifiedAlerting.HAPushPullInterval) + require.Equal(t, alertingDefaultInitializationTimeout, cfg.UnifiedAlerting.InitializationTimeout) } // With peers set, it correctly parses them. @@ -34,10 +35,13 @@ func TestCfg_ReadUnifiedAlertingSettings(t *testing.T) { require.NoError(t, err) _, err = s.NewKey("ha_peers", "hostname1:9090,hostname2:9090,hostname3:9090") require.NoError(t, err) + _, err = s.NewKey("initialization_timeout", "123s") + require.NoError(t, err) require.NoError(t, cfg.ReadUnifiedAlertingSettings(cfg.Raw)) require.Len(t, cfg.UnifiedAlerting.HAPeers, 3) require.ElementsMatch(t, []string{"hostname1:9090", "hostname2:9090", "hostname3:9090"}, cfg.UnifiedAlerting.HAPeers) + require.Equal(t, 123*time.Second, cfg.UnifiedAlerting.InitializationTimeout) } t.Run("should read 'scheduler_tick_interval'", func(t *testing.T) {