Alerting: Add state label to prometheus_imported_rules metric (#106365)

pkg/services/ngalert/metrics/scheduler.go

@@ -200,7 +200,7 @@ func NewSchedulerMetrics(r prometheus.Registerer) *Scheduler {
 				Name:      "prometheus_imported_rules",
 				Help:      "The number of rules imported from a Prometheus-compatible source.",
 			},
-			[]string{"org"},
+			[]string{"org", "state"},
 		),
 	}
 }

pkg/services/ngalert/schedule/metrics.go

@@ -47,7 +47,7 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
 	// gauge for groups per org
 	groupsPerOrg := make(map[int64]map[string]struct{})
 	// gauge for rules imported from Prometheus per org
-	orgsRulesPrometheusImported := make(map[int64]int64)
+	orgsRulesPrometheusImported := make(map[int64]map[string]int64)
 
 	simplifiedEditorSettingsPerOrg := make(map[int64]map[string]int64) // orgID -> setting -> count
 
@@ -89,7 +89,14 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
 		}
 
 		if rule.ImportedPrometheusRule() {
-			orgsRulesPrometheusImported[rule.OrgID]++
+			if orgsRulesPrometheusImported[rule.OrgID] == nil {
+				orgsRulesPrometheusImported[rule.OrgID] = make(map[string]int64)
+			}
+			state := metrics.AlertRuleActiveLabelValue
+			if rule.IsPaused {
+				state = metrics.AlertRulePausedLabelValue
+			}
+			orgsRulesPrometheusImported[rule.OrgID][state]++
 		}
 
 		// Count groups per org
@@ -118,8 +125,10 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
 	for orgID, groups := range groupsPerOrg {
 		sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groups)))
 	}
-	for orgID, count := range orgsRulesPrometheusImported {
-		sch.metrics.PrometheusImportedRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(count))
+	for orgID, counts := range orgsRulesPrometheusImported {
+		for state, count := range counts {
+			sch.metrics.PrometheusImportedRules.WithLabelValues(fmt.Sprint(orgID), state).Set(float64(count))
+		}
 	}
 	for orgID, settings := range simplifiedEditorSettingsPerOrg {
 		for setting, count := range settings {

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -741,13 +741,20 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 			models.RuleGen.WithLabel(models.ConvertedPrometheusRuleLabel, "true"),
 		).GenerateRef()
 
+		alertRulePaused := models.RuleGen.With(
+			models.RuleGen.WithOrgID(firstOrgID),
+			models.RuleGen.WithPrometheusOriginalRuleDefinition("1"),
+			models.RuleGen.WithIsPaused(true),
+		).GenerateRef()
+
 		t.Run("it should show two imported rules in a single org", func(t *testing.T) {
-			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2})
+			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRulePaused})
 
 			expectedMetric := fmt.Sprintf(
 				`# HELP grafana_alerting_prometheus_imported_rules The number of rules imported from a Prometheus-compatible source.
 								# TYPE grafana_alerting_prometheus_imported_rules gauge
-								grafana_alerting_prometheus_imported_rules{org="%[1]d"} 2
+								grafana_alerting_prometheus_imported_rules{org="%[1]d",state="active"} 2
+								grafana_alerting_prometheus_imported_rules{org="%[1]d",state="paused"} 1
 				`, alertRule1.OrgID)
 
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_prometheus_imported_rules")
@@ -760,13 +767,14 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 		).GenerateRef()
 
 		t.Run("it should show three imported rules in two orgs", func(t *testing.T) {
-			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRule3})
+			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2, alertRule3, alertRulePaused})
 
 			expectedMetric := fmt.Sprintf(
 				`# HELP grafana_alerting_prometheus_imported_rules The number of rules imported from a Prometheus-compatible source.
 								# TYPE grafana_alerting_prometheus_imported_rules gauge
-								grafana_alerting_prometheus_imported_rules{org="%[1]d"} 2
-								grafana_alerting_prometheus_imported_rules{org="%[2]d"} 1
+								grafana_alerting_prometheus_imported_rules{org="%[1]d",state="active"} 2
+								grafana_alerting_prometheus_imported_rules{org="%[1]d",state="paused"} 1
+								grafana_alerting_prometheus_imported_rules{org="%[2]d",state="active"} 1
 				`, firstOrgID, secondOrgID)
 
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_prometheus_imported_rules")