diff --git a/pkg/services/ngalert/metrics/alertmanager.go b/pkg/services/ngalert/metrics/alertmanager.go index 58b4630b043..5461e5cc87f 100644 --- a/pkg/services/ngalert/metrics/alertmanager.go +++ b/pkg/services/ngalert/metrics/alertmanager.go @@ -24,6 +24,7 @@ func NewAlertmanagerMetrics(r prometheus.Registerer) *Alertmanager { } type AlertmanagerConfigMetrics struct { + ConfigHash *prometheus.GaugeVec Matchers prometheus.Gauge MatchRE prometheus.Gauge Match prometheus.Gauge @@ -32,6 +33,10 @@ type AlertmanagerConfigMetrics struct { func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMetrics { m := &AlertmanagerConfigMetrics{ + ConfigHash: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "alertmanager_config_hash", + Help: "The hash of the Alertmanager configuration.", + }, []string{"org"}), Matchers: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "alertmanager_config_matchers", Help: "The total number of matchers", @@ -50,7 +55,7 @@ func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMe }), } if r != nil { - r.MustRegister(m.Matchers, m.MatchRE, m.Match, m.ObjectMatchers) + r.MustRegister(m.ConfigHash, m.Matchers, m.MatchRE, m.Match, m.ObjectMatchers) } return m } diff --git a/pkg/services/ngalert/metrics/multi_org_alertmanager.go b/pkg/services/ngalert/metrics/multi_org_alertmanager.go index 150019f774d..94c981c4f12 100644 --- a/pkg/services/ngalert/metrics/multi_org_alertmanager.go +++ b/pkg/services/ngalert/metrics/multi_org_alertmanager.go @@ -117,6 +117,8 @@ type AlertmanagerAggregatedMetrics struct { matchRE *prometheus.Desc match *prometheus.Desc objectMatchers *prometheus.Desc + + configHash *prometheus.Desc } func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *AlertmanagerAggregatedMetrics { @@ -253,6 +255,11 @@ func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *Ale fmt.Sprintf("%s_%s_alertmanager_config_object_matchers", Namespace, Subsystem), "The total number of object_matchers", nil, nil), + + configHash: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_hash", Namespace, Subsystem), + "The hash of the Alertmanager configuration.", + []string{"org"}, nil), } return aggregatedMetrics @@ -296,6 +303,8 @@ func (a *AlertmanagerAggregatedMetrics) Describe(out chan<- *prometheus.Desc) { out <- a.matchRE out <- a.match out <- a.objectMatchers + + out <- a.configHash } func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { @@ -338,4 +347,6 @@ func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfGauges(out, a.matchRE, "alertmanager_config_match_re") data.SendSumOfGauges(out, a.match, "alertmanager_config_match") data.SendSumOfGauges(out, a.objectMatchers, "alertmanager_config_object_matchers") + + data.SendMaxOfGaugesPerTenant(out, a.configHash, "alertmanager_config_hash") } diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index db5a9927d39..887baf85de2 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -3,6 +3,7 @@ package notifier import ( "context" "crypto/md5" + "encoding/binary" "encoding/json" "fmt" "path/filepath" @@ -255,6 +256,10 @@ func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig) { am.ConfigMetrics.MatchRE.Set(float64(amu.MatchRE)) am.ConfigMetrics.Match.Set(float64(amu.Match)) am.ConfigMetrics.ObjectMatchers.Set(float64(amu.ObjectMatchers)) + + am.ConfigMetrics.ConfigHash. + WithLabelValues(strconv.FormatInt(am.orgID, 10)). + Set(hashAsMetricValue(am.Base.ConfigHash())) } func (am *alertmanager) aggregateRouteMatchers(r *apimodels.Route, amu *AggregateMatchersUsage) { @@ -315,8 +320,6 @@ func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig return false, nil } - am.updateConfigMetrics(cfg) - err = am.Base.ApplyConfig(AlertingConfiguration{ rawAlertmanagerConfig: rawConfig, alertmanagerConfig: cfg.AlertmanagerConfig, @@ -327,6 +330,7 @@ func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig return false, err } + am.updateConfigMetrics(cfg) return true, nil } @@ -421,3 +425,13 @@ func (e AlertValidationError) Error() string { type nilLimits struct{} func (n nilLimits) MaxNumberOfAggregationGroups() int { return 0 } + +// This function is taken from upstream, modified to take a [16]byte instead of a []byte. +// https://github.com/prometheus/alertmanager/blob/30fa9cd44bc91c0d6adcc9985609bb08a09a127b/config/coordinator.go#L149-L156 +func hashAsMetricValue(data [16]byte) float64 { + // We only want 48 bits as a float64 only has a 53 bit mantissa. + smallSum := data[0:6] + bytes := make([]byte, 8) + copy(bytes, smallSum) + return float64(binary.LittleEndian.Uint64(bytes)) +}