From f085e99d3c0e40d2ce15777fd4a69d7a178eec6c Mon Sep 17 00:00:00 2001 From: George Robinson Date: Thu, 15 Jun 2023 09:18:01 +0100 Subject: [PATCH] Alerting: Add matchers metrics to Alertmanager (#69855) --- pkg/services/ngalert/metrics/alertmanager.go | 39 ++++++++++++++++- .../ngalert/metrics/multi_org_alertmanager.go | 34 +++++++++++++++ pkg/services/ngalert/notifier/alertmanager.go | 43 +++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/metrics/alertmanager.go b/pkg/services/ngalert/metrics/alertmanager.go index 23b85e20723..58b4630b043 100644 --- a/pkg/services/ngalert/metrics/alertmanager.go +++ b/pkg/services/ngalert/metrics/alertmanager.go @@ -10,12 +10,47 @@ import ( type Alertmanager struct { Registerer prometheus.Registerer *metrics.Alerts + *AlertmanagerConfigMetrics } // NewAlertmanagerMetrics creates a set of metrics for the Alertmanager of each organization. func NewAlertmanagerMetrics(r prometheus.Registerer) *Alertmanager { + other := prometheus.WrapRegistererWithPrefix(fmt.Sprintf("%s_%s_", Namespace, Subsystem), r) return &Alertmanager{ - Registerer: r, - Alerts: metrics.NewAlerts("grafana", prometheus.WrapRegistererWithPrefix(fmt.Sprintf("%s_%s_", Namespace, Subsystem), r)), + Registerer: r, + Alerts: metrics.NewAlerts("grafana", other), + AlertmanagerConfigMetrics: NewAlertmanagerConfigMetrics(r), } } + +type AlertmanagerConfigMetrics struct { + Matchers prometheus.Gauge + MatchRE prometheus.Gauge + Match prometheus.Gauge + ObjectMatchers prometheus.Gauge +} + +func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMetrics { + m := &AlertmanagerConfigMetrics{ + Matchers: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "alertmanager_config_matchers", + Help: "The total number of matchers", + }), + MatchRE: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "alertmanager_config_match_re", + Help: "The total number of matche_re", + }), + Match: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "alertmanager_config_match", + Help: "The total number of match", + }), + ObjectMatchers: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "alertmanager_config_object_matchers", + Help: "The total number of object_matchers", + }), + } + if r != nil { + r.MustRegister(m.Matchers, m.MatchRE, m.Match, m.ObjectMatchers) + } + return m +} diff --git a/pkg/services/ngalert/metrics/multi_org_alertmanager.go b/pkg/services/ngalert/metrics/multi_org_alertmanager.go index 1b19324c54f..65999dbb48b 100644 --- a/pkg/services/ngalert/metrics/multi_org_alertmanager.go +++ b/pkg/services/ngalert/metrics/multi_org_alertmanager.go @@ -107,6 +107,13 @@ type AlertmanagerAggregatedMetrics struct { // exported metrics, gathered from Alertmanager Dispatcher dispatchAggrGroups *prometheus.Desc dispatchProcessingDuration *prometheus.Desc + + // added to measure usage of matchers, match_re, match and + // object_matchers + matchers *prometheus.Desc + matchRE *prometheus.Desc + match *prometheus.Desc + objectMatchers *prometheus.Desc } func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *AlertmanagerAggregatedMetrics { @@ -221,6 +228,23 @@ func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *Ale fmt.Sprintf("%s_%s_dispatcher_alert_processing_duration_seconds", Namespace, Subsystem), "Summary of latencies for the processing of alerts.", nil, nil), + + matchers: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_matchers", Namespace, Subsystem), + "The total number of matchers", + nil, nil), + matchRE: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_match_re", Namespace, Subsystem), + "The total number of matchRE", + nil, nil), + match: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_match", Namespace, Subsystem), + "The total number of match", + nil, nil), + objectMatchers: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_object_matchers", Namespace, Subsystem), + "The total number of object_matchers", + nil, nil), } return aggregatedMetrics @@ -257,6 +281,11 @@ func (a *AlertmanagerAggregatedMetrics) Describe(out chan<- *prometheus.Desc) { out <- a.dispatchAggrGroups out <- a.dispatchProcessingDuration + + out <- a.matchers + out <- a.matchRE + out <- a.match + out <- a.objectMatchers } func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { @@ -292,4 +321,9 @@ func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfGauges(out, a.dispatchAggrGroups, "alertmanager_dispatcher_aggregation_groups") data.SendSumOfSummaries(out, a.dispatchProcessingDuration, "alertmanager_dispatcher_alert_processing_duration_seconds") + + data.SendSumOfGauges(out, a.matchers, "alertmanager_config_matchers") + data.SendSumOfGauges(out, a.matchRE, "alertmanager_config_match_re") + data.SendSumOfGauges(out, a.match, "alertmanager_config_match") + data.SendSumOfGauges(out, a.objectMatchers, "alertmanager_config_object_matchers") } diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 32293d6707c..73584b0ec81 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -12,6 +12,7 @@ import ( alertingNotify "github.com/grafana/alerting/notify" "github.com/grafana/alerting/receivers" alertingTemplates "github.com/grafana/alerting/templates" + "github.com/prometheus/alertmanager/config" amv2 "github.com/prometheus/alertmanager/api/v2/models" @@ -47,6 +48,7 @@ type Alertmanager struct { Base *alertingNotify.GrafanaAlertmanager logger log.Logger + ConfigMetrics *metrics.AlertmanagerConfigMetrics Settings *setting.Cfg Store AlertingStore fileStore *FileStore @@ -133,6 +135,7 @@ func newAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A am := &Alertmanager{ Base: gam, + ConfigMetrics: m.AlertmanagerConfigMetrics, Settings: cfg, Store: store, NotificationService: ns, @@ -237,6 +240,44 @@ func (am *Alertmanager) ApplyConfig(ctx context.Context, dbCfg *ngmodels.AlertCo return outerErr } +type AggregateMatchersUsage struct { + Matchers int + MatchRE int + Match int + ObjectMatchers int +} + +func (am *Alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig) { + var amu AggregateMatchersUsage + am.aggregateRouteMatchers(cfg.AlertmanagerConfig.Route, &amu) + am.aggregateInhibitMatchers(cfg.AlertmanagerConfig.InhibitRules, &amu) + am.ConfigMetrics.Matchers.Set(float64(amu.Matchers)) + am.ConfigMetrics.MatchRE.Set(float64(amu.MatchRE)) + am.ConfigMetrics.Match.Set(float64(amu.Match)) + am.ConfigMetrics.ObjectMatchers.Set(float64(amu.ObjectMatchers)) +} + +func (am *Alertmanager) aggregateRouteMatchers(r *apimodels.Route, amu *AggregateMatchersUsage) { + amu.Matchers += len(r.Matchers) + amu.MatchRE += len(r.MatchRE) + amu.Match += len(r.Match) + amu.ObjectMatchers += len(r.ObjectMatchers) + for _, next := range r.Routes { + am.aggregateRouteMatchers(next, amu) + } +} + +func (am *Alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu *AggregateMatchersUsage) { + for _, r := range rules { + amu.Matchers += len(r.SourceMatchers) + amu.Matchers += len(r.TargetMatchers) + amu.MatchRE += len(r.SourceMatchRE) + amu.MatchRE += len(r.TargetMatchRE) + amu.Match += len(r.SourceMatch) + amu.Match += len(r.TargetMatch) + } +} + // applyConfig applies a new configuration by re-initializing all components using the configuration provided. // It returns a boolean indicating whether the user config was changed and an error. // It is not safe to call concurrently. @@ -274,6 +315,8 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig return false, nil } + am.updateConfigMetrics(cfg) + err = am.Base.ApplyConfig(AlertingConfiguration{ rawAlertmanagerConfig: rawConfig, alertmanagerConfig: cfg.AlertmanagerConfig,