From 539d363d2ce34f7e4985849b9ee96a0eb247da17 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Fri, 6 Sep 2024 18:25:16 +0200 Subject: [PATCH] Create histogram and observe grafana config size (#93028) --- pkg/services/ngalert/metrics/alertmanager.go | 17 +++++++++++------ .../ngalert/metrics/multi_org_alertmanager.go | 10 +++++++++- pkg/services/ngalert/notifier/alertmanager.go | 11 ++++++++--- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pkg/services/ngalert/metrics/alertmanager.go b/pkg/services/ngalert/metrics/alertmanager.go index 2ae5bae5210..597694c9785 100644 --- a/pkg/services/ngalert/metrics/alertmanager.go +++ b/pkg/services/ngalert/metrics/alertmanager.go @@ -24,11 +24,12 @@ func NewAlertmanagerMetrics(r prometheus.Registerer) *Alertmanager { } type AlertmanagerConfigMetrics struct { - ConfigHash *prometheus.GaugeVec - Matchers prometheus.Gauge - MatchRE prometheus.Gauge - Match prometheus.Gauge - ObjectMatchers prometheus.Gauge + ConfigHash *prometheus.GaugeVec + ConfigSizeBytes *prometheus.GaugeVec + Matchers prometheus.Gauge + MatchRE prometheus.Gauge + Match prometheus.Gauge + ObjectMatchers prometheus.Gauge } func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMetrics { @@ -37,6 +38,10 @@ func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMe Name: "alertmanager_config_hash", Help: "The hash of the Alertmanager configuration.", }, []string{"org"}), + ConfigSizeBytes: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "alertmanager_config_size_bytes", + Help: "The size of the grafana alertmanager configuration in bytes", + }, []string{"org"}), Matchers: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "alertmanager_config_matchers", Help: "The total number of matchers", @@ -55,7 +60,7 @@ func NewAlertmanagerConfigMetrics(r prometheus.Registerer) *AlertmanagerConfigMe }), } if r != nil { - r.MustRegister(m.ConfigHash, m.Matchers, m.MatchRE, m.Match, m.ObjectMatchers) + r.MustRegister(m.ConfigHash, m.ConfigSizeBytes, m.Matchers, m.MatchRE, m.Match, m.ObjectMatchers) } return m } diff --git a/pkg/services/ngalert/metrics/multi_org_alertmanager.go b/pkg/services/ngalert/metrics/multi_org_alertmanager.go index 7ed927c901f..3a9cf21f798 100644 --- a/pkg/services/ngalert/metrics/multi_org_alertmanager.go +++ b/pkg/services/ngalert/metrics/multi_org_alertmanager.go @@ -22,7 +22,7 @@ type MultiOrgAlertmanager struct { } func NewMultiOrgAlertmanagerMetrics(r prometheus.Registerer) *MultiOrgAlertmanager { - registries := metrics.NewTenantRegistries(log.New("ngalert.multiorg.alertmanager.metrics")) //TODO: Should this be here? Probably not. + registries := metrics.NewTenantRegistries(log.New("ngalert.multiorg.alertmanager.metrics")) // TODO: Should this be here? Probably not. moa := &MultiOrgAlertmanager{ Registerer: r, registries: registries, @@ -120,6 +120,7 @@ type AlertmanagerAggregatedMetrics struct { objectMatchers *prometheus.Desc configHash *prometheus.Desc + configSize *prometheus.Desc } func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *AlertmanagerAggregatedMetrics { @@ -265,6 +266,11 @@ func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *Ale fmt.Sprintf("%s_%s_alertmanager_config_hash", Namespace, Subsystem), "The hash of the Alertmanager configuration.", []string{"org"}, nil), + + configSize: prometheus.NewDesc( + fmt.Sprintf("%s_%s_alertmanager_config_size_bytes", Namespace, Subsystem), + "The size of the grafana alertmanager configuration in bytes", + []string{"org"}, nil), } return aggregatedMetrics @@ -311,6 +317,7 @@ func (a *AlertmanagerAggregatedMetrics) Describe(out chan<- *prometheus.Desc) { out <- a.objectMatchers out <- a.configHash + out <- a.configSize } func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { @@ -356,4 +363,5 @@ func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfGauges(out, a.objectMatchers, "alertmanager_config_object_matchers") data.SendMaxOfGaugesPerTenant(out, a.configHash, "alertmanager_config_hash") + data.SendMaxOfGaugesPerTenant(out, a.configSize, "alertmanager_config_size_bytes") } diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 51fecf85b21..ad9e74cd811 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -91,7 +91,8 @@ func (m maintenanceOptions) MaintenanceFunc(state alertingNotify.State) (int64, func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store AlertingStore, stateStore stateStore, peer alertingNotify.ClusterPeer, decryptFn alertingNotify.GetDecryptedValueFn, ns notifications.Service, - m *metrics.Alertmanager, withAutogen bool) (*alertmanager, error) { + m *metrics.Alertmanager, withAutogen bool, +) (*alertmanager, error) { nflog, err := stateStore.GetNotificationLog(ctx) if err != nil { return nil, err @@ -283,7 +284,7 @@ type AggregateMatchersUsage struct { ObjectMatchers int } -func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig) { +func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig, cfgSize int) { var amu AggregateMatchersUsage am.aggregateRouteMatchers(cfg.AlertmanagerConfig.Route, &amu) am.aggregateInhibitMatchers(cfg.AlertmanagerConfig.InhibitRules, &amu) @@ -295,6 +296,10 @@ func (am *alertmanager) updateConfigMetrics(cfg *apimodels.PostableUserConfig) { am.ConfigMetrics.ConfigHash. WithLabelValues(strconv.FormatInt(am.orgID, 10)). Set(hashAsMetricValue(am.Base.ConfigHash())) + + am.ConfigMetrics.ConfigSizeBytes. + WithLabelValues(strconv.FormatInt(am.orgID, 10)). + Set(float64(cfgSize)) } func (am *alertmanager) aggregateRouteMatchers(r *apimodels.Route, amu *AggregateMatchersUsage) { @@ -352,7 +357,7 @@ func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig) (bool, er return false, err } - am.updateConfigMetrics(cfg) + am.updateConfigMetrics(cfg, len(rawConfig)) return true, nil }