Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager (#96251)

* Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager

* fix drone image name

pkg/services/ngalert/sender/sender.go

@@ -37,8 +37,9 @@ type ExternalAlertmanager struct {
 
 	manager *Manager
 
-	sdCancel  context.CancelFunc
-	sdManager *discovery.Manager
+	sanitizeLabelSetFn func(lbls models.LabelSet) labels.Labels
+	sdCancel           context.CancelFunc
+	sdManager          *discovery.Manager
 }
 
 type ExternalAMcfg struct {
@@ -57,6 +58,20 @@ func WithDoFunc(doFunc doFunc) Option {
 	}
 }
 
+// WithUTF8Labels skips sanitizing labels and annotations before sending alerts to the external Alertmanager(s).
+// It assumes UTF-8 label names are supported by the Alertmanager(s).
+func WithUTF8Labels() Option {
+	return func(s *ExternalAlertmanager) {
+		s.sanitizeLabelSetFn = func(lbls models.LabelSet) labels.Labels {
+			ls := make(labels.Labels, 0, len(lbls))
+			for k, v := range lbls {
+				ls = append(ls, labels.Label{Name: k, Value: v})
+			}
+			return ls
+		}
+	}
+}
+
 func (cfg *ExternalAMcfg) SHA256() string {
 	return asSHA256([]string{cfg.headerString(), cfg.URL})
 }
@@ -87,6 +102,7 @@ func NewExternalAlertmanagerSender(l log.Logger, reg prometheus.Registerer, opts
 		sdCancel: sdCancel,
 	}
 
+	s.sanitizeLabelSetFn = s.sanitizeLabelSet
 	s.manager = NewManager(
 		// Injecting a new registry here means these metrics are not exported.
 		// Once we fix the individual Alertmanager metrics we should fix this scenario too.
@@ -240,8 +256,8 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str
 func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *Alert {
 	// Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys.
 	return &Alert{
-		Labels:       s.sanitizeLabelSet(alert.Alert.Labels),
-		Annotations:  s.sanitizeLabelSet(alert.Annotations),
+		Labels:       s.sanitizeLabelSetFn(alert.Alert.Labels),
+		Annotations:  s.sanitizeLabelSetFn(alert.Annotations),
 		StartsAt:     time.Time(alert.StartsAt),
 		EndsAt:       time.Time(alert.EndsAt),
 		GeneratorURL: alert.Alert.GeneratorURL.String(),