diff --git a/pkg/services/ngalert/CHANGELOG.md b/pkg/services/ngalert/CHANGELOG.md index 5169d5406fd..83c5c9dd0d5 100644 --- a/pkg/services/ngalert/CHANGELOG.md +++ b/pkg/services/ngalert/CHANGELOG.md @@ -54,7 +54,12 @@ Scopes must have an order to ensure consistency and ease of search, this helps u - [ENHANCEMENT] Create folder 'General Alerting' when Grafana starts from the scratch #48866 - [ENHANCEMENT] Rule changes authorization logic to use UID folder scope instead of ID scope #48970 - [ENHANCEMENT] Scheduler: ticker to support stopping #48142 -- [ENHANCEMENT] Scheduler: Adds new metrics to track rules that might be scheduled. +- [ENHANCEMENT] Scheduler: Adds new metrics to track rules that might be scheduled #49874 + - `grafana_alerting_schedule_alert_rules ` + - `grafana_alerting_schedule_alert_rules_hash ` +- [CHANGE] Scheduler: Renaming of metrics to make them consistent with similar metrics exposed by the component #49874 + - `grafana_alerting_get_alert_rules_duration_seconds` to `grafana_alerting_schedule_periodic_duration_seconds` + - `grafana_alerting_schedule_periodic_duration_seconds` to `grafana_alerting_schedule_query_alert_rules_duration_seconds` - [FEATURE] Indicate whether routes are provisioned when GETting Alertmanager configuration #47857 - [FEATURE] Indicate whether contact point is provisioned when GETting Alertmanager configuration #48323 - [FEATURE] Indicate whether alert rule is provisioned when GETting the rule #48458 diff --git a/pkg/services/ngalert/metrics/ngalert.go b/pkg/services/ngalert/metrics/ngalert.go index 83412deda8b..5fa4fbc2467 100644 --- a/pkg/services/ngalert/metrics/ngalert.go +++ b/pkg/services/ngalert/metrics/ngalert.go @@ -180,7 +180,7 @@ func newSchedulerMetrics(r prometheus.Registerer) *Scheduler { Namespace: Namespace, Subsystem: Subsystem, Name: "schedule_alert_rules", - Help: "The number of alert rules being considered for evaluation each tick.", + Help: "The number of alert rules that could be considered for evaluation at the next tick.", }, ), SchedulableAlertRulesHash: promauto.With(r).NewGauge( @@ -188,7 +188,7 @@ func newSchedulerMetrics(r prometheus.Registerer) *Scheduler { Namespace: Namespace, Subsystem: Subsystem, Name: "schedule_alert_rules_hash", - Help: "A hash of the alert rules over time.", + Help: "A hash of the alert rules that could be considered for evaluation at the next tick.", }), UpdateSchedulableAlertRulesDuration: promauto.With(r).NewHistogram( prometheus.HistogramOpts{ diff --git a/pkg/services/ngalert/schedule/fetcher.go b/pkg/services/ngalert/schedule/fetcher.go index 1175ad4b27d..9db18218a9b 100644 --- a/pkg/services/ngalert/schedule/fetcher.go +++ b/pkg/services/ngalert/schedule/fetcher.go @@ -50,7 +50,5 @@ func (sch *schedule) updateSchedulableAlertRules(ctx context.Context, disabledOr return fmt.Errorf("failed to get alert rules: %w", err) } sch.schedulableAlertRules.set(q.Result) - sch.metrics.SchedulableAlertRules.Set(float64(len(q.Result))) - sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(q.Result))) return nil } diff --git a/pkg/services/ngalert/schedule/registry.go b/pkg/services/ngalert/schedule/registry.go index 9a267917dd5..4ac5a667bb2 100644 --- a/pkg/services/ngalert/schedule/registry.go +++ b/pkg/services/ngalert/schedule/registry.go @@ -151,6 +151,9 @@ func (r *schedulableAlertRulesRegistry) update(rule *models.SchedulableAlertRule r.rules[rule.GetKey()] = rule } +// del removes pair that has specific key from schedulableAlertRulesRegistry. +// Returns 2-tuple where the first element is value of the removed pair +// and the second element indicates whether element with the specified key existed. func (r *schedulableAlertRulesRegistry) del(k models.AlertRuleKey) (*models.SchedulableAlertRule, bool) { r.mu.Lock() defer r.mu.Unlock() diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index 33be349c713..192ac0377df 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -336,6 +336,11 @@ func (sch *schedule) DeleteAlertRule(key models.AlertRuleKey) { } // stop rule evaluation ruleInfo.stop() + + // Our best bet at this point is that we update the metrics with what we hope to schedule in the next tick. + alertRules := sch.schedulableAlertRules.all() + sch.metrics.SchedulableAlertRules.Set(float64(len(alertRules))) + sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(alertRules))) } func (sch *schedule) adminConfigSync(ctx context.Context) error { @@ -390,6 +395,11 @@ func (sch *schedule) schedulePeriodic(ctx context.Context) error { // so, at the end, the remaining registered alert rules are the deleted ones registeredDefinitions := sch.registry.keyMap() + // While these are the rules that we iterate over, at the moment there's no 100% guarantee that they'll be + // scheduled as rules could be removed before we get a chance to evaluate them. + sch.metrics.SchedulableAlertRules.Set(float64(len(alertRules))) + sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(alertRules))) + type readyToRunItem struct { key models.AlertRuleKey ruleInfo *alertRuleInfo