From 82549ea8b31d9ffe7da889997076fefac08f0dbe Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Thu, 5 Jun 2025 14:24:48 +0200 Subject: [PATCH] Alerting: Add state label to prometheus_imported_rules metric (#106365) --- pkg/services/ngalert/metrics/scheduler.go | 2 +- pkg/services/ngalert/schedule/metrics.go | 17 +++++++++++++---- .../ngalert/schedule/schedule_unit_test.go | 18 +++++++++++++----- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pkg/services/ngalert/metrics/scheduler.go b/pkg/services/ngalert/metrics/scheduler.go index fb28e273ebb..2a3cf441702 100644 --- a/pkg/services/ngalert/metrics/scheduler.go +++ b/pkg/services/ngalert/metrics/scheduler.go @@ -200,7 +200,7 @@ func NewSchedulerMetrics(r prometheus.Registerer) *Scheduler { Name: "prometheus_imported_rules", Help: "The number of rules imported from a Prometheus-compatible source.", }, - []string{"org"}, + []string{"org", "state"}, ), } } diff --git a/pkg/services/ngalert/schedule/metrics.go b/pkg/services/ngalert/schedule/metrics.go index 18be3ecb174..e3281c33710 100644 --- a/pkg/services/ngalert/schedule/metrics.go +++ b/pkg/services/ngalert/schedule/metrics.go @@ -47,7 +47,7 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { // gauge for groups per org groupsPerOrg := make(map[int64]map[string]struct{}) // gauge for rules imported from Prometheus per org - orgsRulesPrometheusImported := make(map[int64]int64) + orgsRulesPrometheusImported := make(map[int64]map[string]int64) simplifiedEditorSettingsPerOrg := make(map[int64]map[string]int64) // orgID -> setting -> count @@ -89,7 +89,14 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { } if rule.ImportedPrometheusRule() { - orgsRulesPrometheusImported[rule.OrgID]++ + if orgsRulesPrometheusImported[rule.OrgID] == nil { + orgsRulesPrometheusImported[rule.OrgID] = make(map[string]int64) + } + state := metrics.AlertRuleActiveLabelValue + if rule.IsPaused { + state = metrics.AlertRulePausedLabelValue + } + orgsRulesPrometheusImported[rule.OrgID][state]++ } // Count groups per org @@ -118,8 +125,10 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { for orgID, groups := range groupsPerOrg { sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groups))) } - for orgID, count := range orgsRulesPrometheusImported { - sch.metrics.PrometheusImportedRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(count)) + for orgID, counts := range orgsRulesPrometheusImported { + for state, count := range counts { + sch.metrics.PrometheusImportedRules.WithLabelValues(fmt.Sprint(orgID), state).Set(float64(count)) + } } for orgID, settings := range simplifiedEditorSettingsPerOrg { for setting, count := range settings { diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 8b67ba0ac1b..df249d9e8cd 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -741,13 +741,20 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { models.RuleGen.WithLabel(models.ConvertedPrometheusRuleLabel, "true"), ).GenerateRef() + alertRulePaused := models.RuleGen.With( + models.RuleGen.WithOrgID(firstOrgID), + models.RuleGen.WithPrometheusOriginalRuleDefinition("1"), + models.RuleGen.WithIsPaused(true), + ).GenerateRef() + t.Run("it should show two imported rules in a single org", func(t *testing.T) { - sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2}) + sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRulePaused}) expectedMetric := fmt.Sprintf( `# HELP grafana_alerting_prometheus_imported_rules The number of rules imported from a Prometheus-compatible source. # TYPE grafana_alerting_prometheus_imported_rules gauge - grafana_alerting_prometheus_imported_rules{org="%[1]d"} 2 + grafana_alerting_prometheus_imported_rules{org="%[1]d",state="active"} 2 + grafana_alerting_prometheus_imported_rules{org="%[1]d",state="paused"} 1 `, alertRule1.OrgID) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_prometheus_imported_rules") @@ -760,13 +767,14 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { ).GenerateRef() t.Run("it should show three imported rules in two orgs", func(t *testing.T) { - sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRule3}) + sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRule3, alertRulePaused}) expectedMetric := fmt.Sprintf( `# HELP grafana_alerting_prometheus_imported_rules The number of rules imported from a Prometheus-compatible source. # TYPE grafana_alerting_prometheus_imported_rules gauge - grafana_alerting_prometheus_imported_rules{org="%[1]d"} 2 - grafana_alerting_prometheus_imported_rules{org="%[2]d"} 1 + grafana_alerting_prometheus_imported_rules{org="%[1]d",state="active"} 2 + grafana_alerting_prometheus_imported_rules{org="%[1]d",state="paused"} 1 + grafana_alerting_prometheus_imported_rules{org="%[2]d",state="active"} 1 `, firstOrgID, secondOrgID) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_prometheus_imported_rules")