Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager (#96251)

* Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager

* fix drone image name

.drone.yml

@@ -942,8 +942,9 @@ services:
     path: /var/lib/mysql
 - commands:
   - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+    -alertmanager.utf8-strict-mode-enabled
   environment: {}
-  image: grafana/mimir-alpine:r304-3872ccb
+  image: grafana/mimir-alpine:r316-55f47f8
   name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
@@ -1383,8 +1384,9 @@ services:
     path: /var/lib/mysql
 - commands:
   - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+    -alertmanager.utf8-strict-mode-enabled
   environment: {}
-  image: grafana/mimir-alpine:r304-3872ccb
+  image: grafana/mimir-alpine:r316-55f47f8
   name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
@@ -2455,8 +2457,9 @@ services:
     path: /var/lib/mysql
 - commands:
   - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+    -alertmanager.utf8-strict-mode-enabled
   environment: {}
-  image: grafana/mimir-alpine:r304-3872ccb
+  image: grafana/mimir-alpine:r316-55f47f8
   name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
@@ -3039,8 +3042,9 @@ services:
     path: /var/lib/mysql
 - commands:
   - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+    -alertmanager.utf8-strict-mode-enabled
   environment: {}
-  image: grafana/mimir-alpine:r304-3872ccb
+  image: grafana/mimir-alpine:r316-55f47f8
   name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
@@ -4978,8 +4982,9 @@ services:
     path: /var/lib/mysql
 - commands:
   - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled
+    -alertmanager.utf8-strict-mode-enabled
   environment: {}
-  image: grafana/mimir-alpine:r304-3872ccb
+  image: grafana/mimir-alpine:r316-55f47f8
   name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
@@ -5451,7 +5456,7 @@ steps:
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine
-  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir-alpine:r304-3872ccb
+  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir-alpine:r316-55f47f8
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine
@@ -5489,7 +5494,7 @@ steps:
   - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack
   - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8
   - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine
-  - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir-alpine:r304-3872ccb
+  - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir-alpine:r316-55f47f8
   - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39
   - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32
   - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine
@@ -5735,6 +5740,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: 250d3c2ae9ddf7cbfd2a9fed01fe81312a882ba09b8f8725f4704629da00a774
+hmac: bb28bcd274c9d2ce724db12659ed6e3dcc461f8a07ae0eb9385a64ca5daad4de
 
 ...

devenv/docker/blocks/mimir_backend/docker-compose.yaml

@@ -1,9 +1,10 @@
   mimir_backend:
-    image: grafana/mimir-alpine:r304-3872ccb
+    image: grafana/mimir-alpine:r316-55f47f8
     container_name: mimir_backend
     command:
       - -target=backend
       - -alertmanager.grafana-alertmanager-compatibility-enabled
+      - -alertmanager.utf8-strict-mode-enabled
   nginx:
     environment:
       - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx

pkg/services/ngalert/remote/alertmanager.go

@@ -150,7 +150,12 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
 		return c.Do(req.WithContext(ctx))
 	}
 	senderLogger := log.New("ngalert.sender.external-alertmanager")
-	s, err := sender.NewExternalAlertmanagerSender(senderLogger, prometheus.NewRegistry(), sender.WithDoFunc(doFunc))
+	s, err := sender.NewExternalAlertmanagerSender(
+		senderLogger,
+		prometheus.NewRegistry(),
+		sender.WithDoFunc(doFunc),
+		sender.WithUTF8Labels(),
+	)
 	if err != nil {
 		return nil, err
 	}

pkg/services/ngalert/remote/alertmanager_test.go

@@ -785,16 +785,18 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 	require.Equal(t, 0, len(alertGroups))
 
 	// Let's create two active alerts and one expired one.
-	alert1 := genAlert(true, map[string]string{"test_1": "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"})
+	// UTF-8 label names should be preserved.
-	alert2 := genAlert(true, map[string]string{"test_2": "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"})
+	utf8LabelName := "test utf-8 label 😳"
-	alert3 := genAlert(false, map[string]string{"test_3": "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"})
+	alert1 := genAlert(true, map[string]string{utf8LabelName: "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"})
+	alert2 := genAlert(true, map[string]string{utf8LabelName: "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"})
+	alert3 := genAlert(false, map[string]string{utf8LabelName: "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"})
 	postableAlerts := apimodels.PostableAlerts{
 		PostableAlerts: []amv2.PostableAlert{alert1, alert2, alert3},
 	}
 	err = am.PutAlerts(context.Background(), postableAlerts)
 	require.NoError(t, err)
 
-	// We should have two alerts and one group now.
+	// We should eventually have two active alerts.
 	require.Eventually(t, func() bool {
 		alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{}, "")
 		require.NoError(t, err)
@@ -806,15 +808,15 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 	require.Equal(t, 1, len(alertGroups))
 
 	// Labels with empty values and the namespace UID label should be removed.
+	// UTF-8 label names should remain unchanged.
 	for _, a := range alertGroups {
-		require.NotContains(t, a.Labels, "empty")
+		require.Len(t, a.Alerts, 2)
-		require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel)
+		for _, a := range a.Alerts {
+			require.NotContains(t, a.Labels, "empty")
+			require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel)
+			require.Contains(t, a.Labels, utf8LabelName)
+		}
 	}
-
-	// Filtering by `test_1=test_1` should return one alert.
-	alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{"test_1=test_1"}, "")
-	require.NoError(t, err)
-	require.Equal(t, 1, len(alerts))
 }
 
 func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {

pkg/services/ngalert/sender/sender.go

@@ -37,8 +37,9 @@ type ExternalAlertmanager struct {
 
 	manager *Manager
 
-	sdCancel  context.CancelFunc
+	sanitizeLabelSetFn func(lbls models.LabelSet) labels.Labels
-	sdManager *discovery.Manager
+	sdCancel           context.CancelFunc
+	sdManager          *discovery.Manager
 }
 
 type ExternalAMcfg struct {
@@ -57,6 +58,20 @@ func WithDoFunc(doFunc doFunc) Option {
 	}
 }
 
+// WithUTF8Labels skips sanitizing labels and annotations before sending alerts to the external Alertmanager(s).
+// It assumes UTF-8 label names are supported by the Alertmanager(s).
+func WithUTF8Labels() Option {
+	return func(s *ExternalAlertmanager) {
+		s.sanitizeLabelSetFn = func(lbls models.LabelSet) labels.Labels {
+			ls := make(labels.Labels, 0, len(lbls))
+			for k, v := range lbls {
+				ls = append(ls, labels.Label{Name: k, Value: v})
+			}
+			return ls
+		}
+	}
+}
+
 func (cfg *ExternalAMcfg) SHA256() string {
 	return asSHA256([]string{cfg.headerString(), cfg.URL})
 }
@@ -87,6 +102,7 @@ func NewExternalAlertmanagerSender(l log.Logger, reg prometheus.Registerer, opts
 		sdCancel: sdCancel,
 	}
 
+	s.sanitizeLabelSetFn = s.sanitizeLabelSet
 	s.manager = NewManager(
 		// Injecting a new registry here means these metrics are not exported.
 		// Once we fix the individual Alertmanager metrics we should fix this scenario too.
@@ -240,8 +256,8 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str
 func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *Alert {
 	// Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys.
 	return &Alert{
-		Labels:       s.sanitizeLabelSet(alert.Alert.Labels),
+		Labels:       s.sanitizeLabelSetFn(alert.Alert.Labels),
-		Annotations:  s.sanitizeLabelSet(alert.Annotations),
+		Annotations:  s.sanitizeLabelSetFn(alert.Annotations),
 		StartsAt:     time.Time(alert.StartsAt),
 		EndsAt:       time.Time(alert.EndsAt),
 		GeneratorURL: alert.Alert.GeneratorURL.String(),

scripts/drone/services/services.star

@@ -57,7 +57,7 @@ def integration_test_services():
             "name": "mimir_backend",
             "image": images["mimir"],
             "environment": {},
-            "commands": ["/bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled"],
+            "commands": ["/bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled -alertmanager.utf8-strict-mode-enabled"],
         },
         {
             "name": "redis",

scripts/drone/utils/images.star

@@ -22,7 +22,7 @@ images = {
     "plugins_slack": "plugins/slack",
     "python": "python:3.8",
     "postgres_alpine": "postgres:12.3-alpine",
-    "mimir": "grafana/mimir-alpine:r304-3872ccb",
+    "mimir": "grafana/mimir-alpine:r316-55f47f8",
     "mysql5": "mysql:5.7.39",
     "mysql8": "mysql:8.0.32",
     "redis_alpine": "redis:6.2.11-alpine",