public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add config disabled_labels to disable reserved labels (#51832)

Browse Source 
* Alerting: Add config disabled_labels to disable reserved labels

[unified_alerting.reserved_labels]
disabled_labels

* Replace IsGrafanaFolderDisabled with more generic IsReservedLabelDisabled

* Simplify SchedulerCfg by including UnifiedAlertingSettings
This commit is contained in:
Matthew Jacobson
2022-07-11 12:41:40 -04:00
committed by GitHub
parent 434e94ef2b
commit 28dd413c1d
9 changed files with 131 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/ngalert.go
+10 -14
View File
@@ -129,20 +129,16 @@ func (ng *AlertNG) init() error {
}
schedCfg := schedule.SchedulerCfg{
C: clock.New(),
BaseInterval: ng.Cfg.UnifiedAlerting.BaseInterval,
Logger: ng.Log,
MaxAttempts: ng.Cfg.UnifiedAlerting.MaxAttempts,
Evaluator: eval.NewEvaluator(ng.Cfg, ng.Log, ng.DataSourceCache, ng.SecretsService, ng.ExpressionService),
InstanceStore: store,
RuleStore: store,
AdminConfigStore: store,
OrgStore: store,
MultiOrgNotifier: ng.MultiOrgAlertmanager,
Metrics: ng.Metrics.GetSchedulerMetrics(),
AdminConfigPollInterval: ng.Cfg.UnifiedAlerting.AdminConfigPollInterval,
DisabledOrgs: ng.Cfg.UnifiedAlerting.DisabledOrgs,
MinRuleInterval: ng.Cfg.UnifiedAlerting.MinInterval,
Cfg: ng.Cfg.UnifiedAlerting,
C: clock.New(),
Logger: ng.Log,
Evaluator: eval.NewEvaluator(ng.Cfg, ng.Log, ng.DataSourceCache, ng.SecretsService, ng.ExpressionService),
InstanceStore: store,
RuleStore: store,
AdminConfigStore: store,
OrgStore: store,
MultiOrgNotifier: ng.MultiOrgAlertmanager,
Metrics: ng.Metrics.GetSchedulerMetrics(),
}
appUrl, err := url.Parse(ng.Cfg.AppURL)
pkg/services/ngalert/schedule/schedule.go
+39 -35
View File
@@ -21,6 +21,7 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/sender"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/setting"
"github.com/benbjohnson/clock"
"golang.org/x/sync/errgroup"
@@ -89,7 +90,8 @@ type schedule struct {
stateManager *state.Manager
appURL *url.URL
appURL *url.URL
disableGrafanaFolder bool
multiOrgNotifier *notifier.MultiOrgAlertmanager
metrics *metrics.Scheduler
@@ -115,33 +117,29 @@ type schedule struct {
// SchedulerCfg is the scheduler configuration.
type SchedulerCfg struct {
C clock.Clock
BaseInterval time.Duration
Logger log.Logger
EvalAppliedFunc func(ngmodels.AlertRuleKey, time.Time)
MaxAttempts int64
StopAppliedFunc func(ngmodels.AlertRuleKey)
Evaluator eval.Evaluator
RuleStore store.RuleStore
OrgStore store.OrgStore
InstanceStore store.InstanceStore
AdminConfigStore store.AdminConfigurationStore
MultiOrgNotifier *notifier.MultiOrgAlertmanager
Metrics *metrics.Scheduler
AdminConfigPollInterval time.Duration
DisabledOrgs map[int64]struct{}
MinRuleInterval time.Duration
Cfg setting.UnifiedAlertingSettings
C clock.Clock
Logger log.Logger
EvalAppliedFunc func(ngmodels.AlertRuleKey, time.Time)
StopAppliedFunc func(ngmodels.AlertRuleKey)
Evaluator eval.Evaluator
RuleStore store.RuleStore
OrgStore store.OrgStore
InstanceStore store.InstanceStore
AdminConfigStore store.AdminConfigurationStore
MultiOrgNotifier *notifier.MultiOrgAlertmanager
Metrics *metrics.Scheduler
}
// NewScheduler returns a new schedule.
func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager, bus bus.Bus) *schedule {
ticker := alerting.NewTicker(cfg.C, cfg.BaseInterval, cfg.Metrics.Ticker)
ticker := alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
sch := schedule{
registry: alertRuleInfoRegistry{alertRuleInfo: make(map[ngmodels.AlertRuleKey]*alertRuleInfo)},
maxAttempts: cfg.MaxAttempts,
maxAttempts: cfg.Cfg.MaxAttempts,
clock: cfg.C,
baseInterval: cfg.BaseInterval,
baseInterval: cfg.Cfg.BaseInterval,
log: cfg.Logger,
ticker: ticker,
evalAppliedFunc: cfg.EvalAppliedFunc,
@@ -154,13 +152,14 @@ func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager
multiOrgNotifier: cfg.MultiOrgNotifier,
metrics: cfg.Metrics,
appURL: appURL,
disableGrafanaFolder: cfg.Cfg.ReservedLabels.IsReservedLabelDisabled(ngmodels.FolderTitleLabel),
stateManager: stateManager,
sendAlertsTo: map[int64]ngmodels.AlertmanagersChoice{},
senders: map[int64]*sender.Sender{},
sendersCfgHash: map[int64]string{},
adminConfigPollInterval: cfg.AdminConfigPollInterval,
disabledOrgs: cfg.DisabledOrgs,
minRuleInterval: cfg.MinRuleInterval,
adminConfigPollInterval: cfg.Cfg.AdminConfigPollInterval,
disabledOrgs: cfg.Cfg.DisabledOrgs,
minRuleInterval: cfg.Cfg.MinInterval,
schedulableAlertRules: schedulableAlertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.SchedulableAlertRule)},
bus: bus,
}
@@ -600,18 +599,20 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
OrgId: key.OrgID,
}
folder, err := sch.ruleStore.GetNamespaceByUID(ctx, q.Result.NamespaceUID, q.Result.OrgID, user)
if err != nil {
logger.Error("failed to fetch alert rule namespace", "err", err)
return nil, err
}
if !sch.disableGrafanaFolder {
folder, err := sch.ruleStore.GetNamespaceByUID(ctx, q.Result.NamespaceUID, q.Result.OrgID, user)
if err != nil {
logger.Error("failed to fetch alert rule namespace", "err", err)
return nil, err
}
if q.Result.Labels == nil {
q.Result.Labels = make(map[string]string)
} else if val, ok := q.Result.Labels[ngmodels.FolderTitleLabel]; ok {
logger.Warn("alert rule contains protected label, value will be overwritten", "label", ngmodels.FolderTitleLabel, "value", val)
if q.Result.Labels == nil {
q.Result.Labels = make(map[string]string)
} else if val, ok := q.Result.Labels[ngmodels.FolderTitleLabel]; ok {
logger.Warn("alert rule contains protected label, value will be overwritten", "label", ngmodels.FolderTitleLabel, "value", val)
}
q.Result.Labels[ngmodels.FolderTitleLabel] = folder.Title
}
q.Result.Labels[ngmodels.FolderTitleLabel] = folder.Title
return q.Result, nil
}
@@ -740,15 +741,18 @@ func (sch *schedule) saveAlertStates(ctx context.Context, states []*state.State)
// folderUpdateHandler listens for folder update events and updates all rules in the given folder.
func (sch *schedule) folderUpdateHandler(ctx context.Context, evt *events.FolderUpdated) error {
if sch.disableGrafanaFolder {
return nil
}
return sch.UpdateAlertRulesByNamespaceUID(ctx, evt.OrgID, evt.UID)
}
// overrideCfg is only used on tests.
func (sch *schedule) overrideCfg(cfg SchedulerCfg) {
sch.clock = cfg.C
sch.baseInterval = cfg.BaseInterval
sch.baseInterval = cfg.Cfg.BaseInterval
sch.ticker.Stop()
sch.ticker = alerting.NewTicker(cfg.C, cfg.BaseInterval, cfg.Metrics.Ticker)
sch.ticker = alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
sch.evalAppliedFunc = cfg.EvalAppliedFunc
sch.stopAppliedFunc = cfg.StopAppliedFunc
}
pkg/services/ngalert/schedule/schedule_test.go
+26 -19
View File
@@ -27,6 +27,7 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/tests"
"github.com/grafana/grafana/pkg/setting"
)
var testMetrics = metrics.NewNGAlert(prometheus.NewPedanticRegistry())
@@ -98,16 +99,19 @@ func TestWarmStateCache(t *testing.T) {
}
_ = dbstore.SaveAlertInstance(ctx, saveCmd2)
schedCfg := schedule.SchedulerCfg{
C: clock.NewMock(),
BaseInterval: time.Second,
Logger: log.New("ngalert cache warming test"),
RuleStore: dbstore,
InstanceStore: dbstore,
Metrics: testMetrics.GetSchedulerMetrics(),
cfg := setting.UnifiedAlertingSettings{
BaseInterval: time.Second,
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
}
schedCfg := schedule.SchedulerCfg{
Cfg: cfg,
C: clock.NewMock(),
Logger: log.New("ngalert cache warming test"),
RuleStore: dbstore,
InstanceStore: dbstore,
Metrics: testMetrics.GetSchedulerMetrics(),
}
st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
st.Warm(ctx)
@@ -140,25 +144,28 @@ func TestAlertingTicker(t *testing.T) {
stopAppliedCh := make(chan models.AlertRuleKey, len(alerts))
mockedClock := clock.NewMock()
baseInterval := time.Second
cfg := setting.UnifiedAlertingSettings{
BaseInterval: time.Second,
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
DisabledOrgs: map[int64]struct{}{
disabledOrgID: {},
},
}
schedCfg := schedule.SchedulerCfg{
C: mockedClock,
BaseInterval: baseInterval,
Cfg: cfg,
C: mockedClock,
EvalAppliedFunc: func(alertDefKey models.AlertRuleKey, now time.Time) {
evalAppliedCh <- evalAppliedInfo{alertDefKey: alertDefKey, now: now}
},
StopAppliedFunc: func(alertDefKey models.AlertRuleKey) {
stopAppliedCh <- alertDefKey
},
RuleStore: dbstore,
InstanceStore: dbstore,
Logger: log.New("ngalert schedule test"),
Metrics: testMetrics.GetSchedulerMetrics(),
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
DisabledOrgs: map[int64]struct{}{
disabledOrgID: {},
},
RuleStore: dbstore,
InstanceStore: dbstore,
Logger: log.New("ngalert schedule test"),
Metrics: testMetrics.GetSchedulerMetrics(),
}
st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
appUrl := &url.URL{
pkg/services/ngalert/schedule/schedule_unit_test.go
+13 -9
View File
@@ -945,19 +945,23 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}, &notifier.FakeKVStore{}, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
schedCfg := SchedulerCfg{
C: mockedClock,
cfg := setting.UnifiedAlertingSettings{
BaseInterval: time.Second,
MaxAttempts: 1,
Evaluator: eval.NewEvaluator(&setting.Cfg{ExpressionsEnabled: true}, logger, nil, secretsService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil)),
RuleStore: rs,
InstanceStore: is,
AdminConfigStore: acs,
MultiOrgNotifier: moa,
Logger: logger,
Metrics: m.GetSchedulerMetrics(),
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
}
schedCfg := SchedulerCfg{
Cfg: cfg,
C: mockedClock,
Evaluator: eval.NewEvaluator(&setting.Cfg{ExpressionsEnabled: true}, logger, nil, secretsService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil)),
RuleStore: rs,
InstanceStore: is,
AdminConfigStore: acs,
MultiOrgNotifier: moa,
Logger: logger,
Metrics: m.GetSchedulerMetrics(),
}
st := state.NewManager(schedCfg.Logger, m.GetStateMetrics(), nil, rs, is, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
appUrl := &url.URL{
Scheme: "http",
pkg/setting/setting_unified_alerting.go
+20
View File
@@ -82,6 +82,7 @@ type UnifiedAlertingSettings struct {
// DefaultRuleEvaluationInterval default interval between evaluations of a rule.
DefaultRuleEvaluationInterval time.Duration
Screenshots UnifiedAlertingScreenshotSettings
ReservedLabels UnifiedAlertingReservedLabelSettings
}
type UnifiedAlertingScreenshotSettings struct {
@@ -90,12 +91,22 @@ type UnifiedAlertingScreenshotSettings struct {
UploadExternalImageStorage bool
}
type UnifiedAlertingReservedLabelSettings struct {
DisabledLabels map[string]struct{}
}
// IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
// It hides the implementation details of the Enabled and simplifies its usage.
func (u *UnifiedAlertingSettings) IsEnabled() bool {
return u.Enabled == nil || *u.Enabled
}
// IsReservedLabelDisabled returns true if UnifiedAlertingReservedLabelSettings.DisabledLabels contains the given reserved label.
func (u *UnifiedAlertingReservedLabelSettings) IsReservedLabelDisabled(label string) bool {
_, ok := u.DisabledLabels[label]
return ok
}
// readUnifiedAlertingEnabledSettings reads the settings for unified alerting.
// It returns a non-nil bool and a nil error when unified alerting is enabled either
// because it has been enabled in the settings or by default. It returns nil and
@@ -274,6 +285,15 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfgScreenshots.UploadExternalImageStorage = screenshots.Key("upload_external_image_storage").MustBool(screenshotsDefaultUploadImageStorage)
uaCfg.Screenshots = uaCfgScreenshots
reservedLabels := iniFile.Section("unified_alerting.reserved_labels")
uaCfgReservedLabels := UnifiedAlertingReservedLabelSettings{
DisabledLabels: make(map[string]struct{}),
}
for _, label := range util.SplitString(reservedLabels.Key("disabled_labels").MustString("")) {
uaCfgReservedLabels.DisabledLabels[label] = struct{}{}
}
uaCfg.ReservedLabels = uaCfgReservedLabels
cfg.UnifiedAlerting = uaCfg
return nil
}