Alerting: Add limits for the size of expanded notification templates (#115242)
* [create-pull-request] automated change * propagate template limits from config * fmt --------- Co-authored-by: yuri-tceretian <25988953+yuri-tceretian@users.noreply.github.com> Co-authored-by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>
This commit is contained in:
alerting-team[bot]
co-authored by
yuri-tceretian
Yuri Tseretyan
yuri-tceretian
Yuri Tseretyan
parent
fdc84474ce
commit
7779c90713
+1
-1
@@ -149,7 +149,7 @@ require (
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 // indirect
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f // indirect
|
||||
github.com/grafana/dataplane/sdata v0.0.9 // indirect
|
||||
github.com/grafana/dskit v0.0.0-20250908063411-6b6da59b5cc4 // indirect
|
||||
|
||||
+2
-2
@@ -606,8 +606,8 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
|
||||
github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba h1:psKWNETD5nGxmFAlqnWsXoRyUwSa2GHNEMSEDKGKfQ4=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 h1:ZzG/gCclEit9w0QUfQt9GURcOycAIGcsQAhY1u0AEX0=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f h1:Cbm6OKkOcJ+7CSZsGsEJzktC/SIa5bxVeYKQLuYK86o=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f/go.mod h1:axY0cdOg3q0TZHwpHnIz5x16xZ8ZBxJHShsSHHXcHQg=
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 h1:Muoy+FMGrHj3GdFbvsMzUT7eusgii9PKf9L1ZaXDDbY=
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.25.5
|
||||
|
||||
require (
|
||||
github.com/go-kit/log v0.2.1
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7
|
||||
github.com/grafana/dskit v0.0.0-20250908063411-6b6da59b5cc4
|
||||
github.com/grafana/grafana-app-sdk v0.48.5
|
||||
github.com/grafana/grafana-app-sdk/logging v0.48.3
|
||||
|
||||
@@ -216,12 +216,10 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grafana/grafana-app-sdk v0.48.5 h1:MS8l9fTZz+VbTfgApn09jw27GxhQ6fNOWGhC4ydvZmM=
|
||||
github.com/grafana/grafana-app-sdk v0.48.5/go.mod h1:HJsMOSBmt/D/Ihs1SvagOwmXKi0coBMVHlfvdd+qe9Y=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba h1:psKWNETD5nGxmFAlqnWsXoRyUwSa2GHNEMSEDKGKfQ4=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 h1:ZzG/gCclEit9w0QUfQt9GURcOycAIGcsQAhY1u0AEX0=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/dskit v0.0.0-20250908063411-6b6da59b5cc4 h1:jSojuc7njleS3UOz223WDlXOinmuLAIPI0z2vtq8EgI=
|
||||
github.com/grafana/dskit v0.0.0-20250908063411-6b6da59b5cc4/go.mod h1:VahT+GtfQIM+o8ht2StR6J9g+Ef+C2Vokh5uuSmOD/4=
|
||||
github.com/grafana/grafana-app-sdk v0.48.5 h1:MS8l9fTZz+VbTfgApn09jw27GxhQ6fNOWGhC4ydvZmM=
|
||||
|
||||
+1
-1
@@ -221,7 +221,7 @@ require (
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 // indirect
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f // indirect
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 // indirect
|
||||
github.com/grafana/dataplane/sdata v0.0.9 // indirect
|
||||
|
||||
+2
-2
@@ -817,8 +817,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba h1:psKWNETD5nGxmFAlqnWsXoRyUwSa2GHNEMSEDKGKfQ4=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 h1:ZzG/gCclEit9w0QUfQt9GURcOycAIGcsQAhY1u0AEX0=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f h1:Cbm6OKkOcJ+7CSZsGsEJzktC/SIa5bxVeYKQLuYK86o=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f/go.mod h1:axY0cdOg3q0TZHwpHnIz5x16xZ8ZBxJHShsSHHXcHQg=
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 h1:Muoy+FMGrHj3GdFbvsMzUT7eusgii9PKf9L1ZaXDDbY=
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ require (
|
||||
github.com/google/gnostic-models v0.7.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba // indirect
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 // indirect
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f // indirect
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 // indirect
|
||||
github.com/grafana/dataplane/sdata v0.0.9 // indirect
|
||||
|
||||
+2
-2
@@ -174,8 +174,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba h1:psKWNETD5nGxmFAlqnWsXoRyUwSa2GHNEMSEDKGKfQ4=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 h1:ZzG/gCclEit9w0QUfQt9GURcOycAIGcsQAhY1u0AEX0=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f h1:Cbm6OKkOcJ+7CSZsGsEJzktC/SIa5bxVeYKQLuYK86o=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f/go.mod h1:axY0cdOg3q0TZHwpHnIz5x16xZ8ZBxJHShsSHHXcHQg=
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 h1:Muoy+FMGrHj3GdFbvsMzUT7eusgii9PKf9L1ZaXDDbY=
|
||||
|
||||
@@ -1327,6 +1327,10 @@ alertmanager_max_silences_count =
|
||||
# Maximum silence size in bytes. Default: 0 (no limit).
|
||||
alertmanager_max_silence_size_bytes =
|
||||
|
||||
# Maximum size of the expanded template output in bytes. Default: 10485760 (0 - no limit).
|
||||
# The result of template expansion will be truncated to the limit.
|
||||
alertmanager_max_template_output_bytes =
|
||||
|
||||
# Redis server address or addresses. It can be a single Redis address if using Redis standalone,
|
||||
# or a list of comma-separated addresses if using Redis Cluster/Sentinel.
|
||||
ha_redis_address =
|
||||
|
||||
@@ -1776,6 +1776,13 @@ Specify the frequency of polling for Alertmanager configuration changes. The def
|
||||
|
||||
The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), for example, 30s or 1m.
|
||||
|
||||
#### `alertmanager_max_template_output_bytes`
|
||||
|
||||
Maximum size in bytes that the expanded result of any single template expression (e.g. {{ .CommonAnnotations.description }}, {{ .ExternalURL }}, etc.) may reach during notification rendering.
|
||||
The limit is checked after template execution for each templated field, but before the value is inserted into the final notification payload sent to the receiver.
|
||||
If exceeded, the notification will contain output truncated up to the limit and a warning will be logged.
|
||||
The default value is 10,485,760 bytes (10Mb).
|
||||
|
||||
#### `ha_redis_address`
|
||||
|
||||
Redis server address or addresses. It can be a single Redis address if using Redis standalone,
|
||||
|
||||
@@ -87,7 +87,7 @@ require (
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // @grafana/grafana-backend-group
|
||||
github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // @grafana/grafana-app-platform-squad
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba // @grafana/alerting-backend
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 // @grafana/alerting-backend
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f // @grafana/identity-access-team
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 // @grafana/identity-access-team
|
||||
github.com/grafana/dataplane/examples v0.0.1 // @grafana/observability-metrics
|
||||
|
||||
@@ -1613,8 +1613,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
|
||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba h1:psKWNETD5nGxmFAlqnWsXoRyUwSa2GHNEMSEDKGKfQ4=
|
||||
github.com/grafana/alerting v0.0.0-20251204145817-de8c2bbf9eba/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7 h1:ZzG/gCclEit9w0QUfQt9GURcOycAIGcsQAhY1u0AEX0=
|
||||
github.com/grafana/alerting v0.0.0-20251212143239-491433b332b7/go.mod h1:l7v67cgP7x72ajB9UPZlumdrHqNztpKoqQ52cU8T3LU=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f h1:Cbm6OKkOcJ+7CSZsGsEJzktC/SIa5bxVeYKQLuYK86o=
|
||||
github.com/grafana/authlib v0.0.0-20250930082137-a40e2c2b094f/go.mod h1:axY0cdOg3q0TZHwpHnIz5x16xZ8ZBxJHShsSHHXcHQg=
|
||||
github.com/grafana/authlib/types v0.0.0-20251119142549-be091cf2f4d4 h1:Muoy+FMGrHj3GdFbvsMzUT7eusgii9PKf9L1ZaXDDbY=
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/grafana/alerting/models"
|
||||
alertingNotify "github.com/grafana/alerting/notify"
|
||||
"github.com/grafana/alerting/notify/nfstatus"
|
||||
alertingTemplates "github.com/grafana/alerting/templates"
|
||||
"github.com/prometheus/alertmanager/config"
|
||||
|
||||
amv2 "github.com/prometheus/alertmanager/api/v2/models"
|
||||
@@ -58,6 +59,7 @@ type alertmanager struct {
|
||||
decryptFn alertingNotify.GetDecryptedValueFn
|
||||
crypto Crypto
|
||||
features featuremgmt.FeatureToggles
|
||||
dynamicLimits alertingNotify.DynamicLimits
|
||||
}
|
||||
|
||||
// maintenanceOptions represent the options for components that need maintenance on a frequency within the Alertmanager.
|
||||
@@ -148,6 +150,16 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
|
||||
return nil, err
|
||||
}
|
||||
|
||||
limits := alertingNotify.DynamicLimits{
|
||||
Dispatcher: nilLimits{},
|
||||
Templates: alertingTemplates.Limits{
|
||||
MaxTemplateOutputSize: cfg.UnifiedAlerting.AlertmanagerMaxTemplateOutputSize,
|
||||
},
|
||||
}
|
||||
if err := limits.Templates.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid template limits: %w", err)
|
||||
}
|
||||
|
||||
am := &alertmanager{
|
||||
Base: gam,
|
||||
ConfigMetrics: m.AlertmanagerConfigMetrics,
|
||||
@@ -158,6 +170,7 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
|
||||
decryptFn: decryptFn,
|
||||
crypto: crypto,
|
||||
features: featureToggles,
|
||||
dynamicLimits: limits,
|
||||
}
|
||||
|
||||
return am, nil
|
||||
@@ -382,7 +395,7 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable
|
||||
TimeIntervals: amConfig.TimeIntervals,
|
||||
Templates: templates,
|
||||
Receivers: receivers,
|
||||
DispatcherLimits: &nilLimits{},
|
||||
Limits: am.dynamicLimits,
|
||||
Raw: rawConfig,
|
||||
Hash: configHash,
|
||||
})
|
||||
|
||||
@@ -153,6 +153,9 @@ type UnifiedAlertingSettings struct {
|
||||
|
||||
// DeletedRuleRetention defines the maximum duration to retain deleted alerting rules before permanent removal.
|
||||
DeletedRuleRetention time.Duration
|
||||
|
||||
// AlertmanagerMaxTemplateOutputSize specifies the maximum allowed size for rendered template output in bytes.
|
||||
AlertmanagerMaxTemplateOutputSize int64
|
||||
}
|
||||
|
||||
type RecordingRuleSettings struct {
|
||||
@@ -583,6 +586,11 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
|
||||
return fmt.Errorf("setting 'deleted_rule_retention' is invalid, only 0 or a positive duration are allowed")
|
||||
}
|
||||
|
||||
uaCfg.AlertmanagerMaxTemplateOutputSize = ua.Key("alertmanager_max_template_output_bytes").MustInt64(10485760)
|
||||
if uaCfg.AlertmanagerMaxTemplateOutputSize < 0 {
|
||||
return fmt.Errorf("setting 'alertmanager_max_template_output_bytes' is invalid, only 0 or a positive integer are allowed")
|
||||
}
|
||||
|
||||
cfg.UnifiedAlerting = uaCfg
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user