diff --git a/docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md b/docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md index af9959c7c98..fd0e9e51efe 100644 --- a/docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md +++ b/docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md @@ -23,6 +23,20 @@ This topic explains why labels are a fundamental component of alerting. {{< figure src="/static/img/docs/alerting/unified/rule-edit-details-8-0.png" max-width="550px" caption="Alert details" >}} +# External Alertmanager Compatibility + +Grafana's built-in Alertmanager supports both Unicode label keys and values. If you are using an external Prometheus Alertmanager, label keys must be compatible with their [data model](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). +This means that label keys must only contain **ASCII letters**, **numbers**, as well as **underscores** and match the regex `[a-zA-Z_][a-zA-Z0-9_]*`. +Any invalid characters will be removed or replaced by the Grafana alerting engine before being sent to the external Alertmanager according to the following rules: + +- `Whitespace` will be removed. +- `ASCII characters` will be replaced with `_`. +- `All other characters` will be replaced with their lower-case hex representation. If this is the first character it will be prefixed with `_`. + +Example: A label key/value pair `Alert! 🔔="🔥"` will become `Alert_0x1f514="🔥"`. + +**Note** If multiple label keys are sanitized to the same value, the duplicates will have a short hash of the original label appended as a suffix. + # Grafana reserved labels > **Note:** Labels prefixed with `grafana_` are reserved by Grafana for special use. If a manually configured label is added beginning with `grafana_` it may be overwritten in case of collision. diff --git a/pkg/services/ngalert/sender/sender.go b/pkg/services/ngalert/sender/sender.go index d1b8b7c75e5..e2d61204484 100644 --- a/pkg/services/ngalert/sender/sender.go +++ b/pkg/services/ngalert/sender/sender.go @@ -2,10 +2,15 @@ package sender import ( "context" + "crypto/md5" + "errors" + "fmt" "net/url" + "sort" "strings" "sync" "time" + "unicode" "github.com/grafana/grafana/pkg/infra/log" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" @@ -100,7 +105,7 @@ func (s *ExternalAlertmanager) SendAlerts(alerts apimodels.PostableAlerts) { } as := make([]*notifier.Alert, 0, len(alerts.PostableAlerts)) for _, a := range alerts.PostableAlerts { - na := alertToNotifierAlert(a) + na := s.alertToNotifierAlert(a) as = append(as, na) } @@ -171,29 +176,114 @@ func buildNotifierConfig(cfg *ngmodels.AdminConfiguration) (*config.Config, erro return notifierConfig, nil } -func alertToNotifierAlert(alert models.PostableAlert) *notifier.Alert { - ls := make(labels.Labels, 0, len(alert.Alert.Labels)) - a := make(labels.Labels, 0, len(alert.Annotations)) - - // Prometheus does not allow spaces in labels or annotations while Grafana does, we need to make sure we - // remove them before sending the alerts. - for k, v := range alert.Alert.Labels { - ls = append(ls, labels.Label{Name: removeSpaces(k), Value: v}) - } - - for k, v := range alert.Annotations { - a = append(a, labels.Label{Name: removeSpaces(k), Value: v}) - } - +func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *notifier.Alert { + // Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys. return ¬ifier.Alert{ - Labels: ls, - Annotations: a, + Labels: s.sanitizeLabelSet(alert.Alert.Labels), + Annotations: s.sanitizeLabelSet(alert.Annotations), StartsAt: time.Time(alert.StartsAt), EndsAt: time.Time(alert.EndsAt), GeneratorURL: alert.Alert.GeneratorURL.String(), } } -func removeSpaces(labelName string) string { - return strings.Join(strings.Fields(labelName), "") +// sanitizeLabelSet sanitizes all given LabelSet keys according to sanitizeLabelName. +// If there is a collision as a result of sanitization, a short (6 char) md5 hash of the original key will be added as a suffix. +func (s *ExternalAlertmanager) sanitizeLabelSet(lbls models.LabelSet) labels.Labels { + ls := make(labels.Labels, 0, len(lbls)) + set := make(map[string]struct{}) + + // Must sanitize labels in order otherwise resulting label set can be inconsistent when there are collisions. + for _, k := range sortedKeys(lbls) { + sanitizedLabelName, err := s.sanitizeLabelName(k) + if err != nil { + s.logger.Error("alert sending to external Alertmanager(s) contains an invalid label/annotation name that failed to sanitize, skipping", "name", k, "err", err) + continue + } + + // There can be label name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name. + if _, ok := set[sanitizedLabelName]; ok { + sanitizedLabelName = sanitizedLabelName + fmt.Sprintf("_%.3x", md5.Sum([]byte(k))) + s.logger.Warn("alert contains duplicate label/annotation name after sanitization, appending unique suffix", "name", k, "new_name", sanitizedLabelName, "err", err) + } + + set[sanitizedLabelName] = struct{}{} + ls = append(ls, labels.Label{Name: sanitizedLabelName, Value: lbls[k]}) + } + + return ls +} + +// sanitizeLabelName will fix a given label name so that it is compatible with prometheus alertmanager character restrictions. +// Prometheus alertmanager requires labels to match ^[a-zA-Z_][a-zA-Z0-9_]*$. +// Characters with an ASCII code < 127 will be replaced with an underscore (_), characters with ASCII code >= 127 will be replaced by their hex representation. +// For backwards compatibility, whitespace will be removed instead of replaced with an underscore. +func (s *ExternalAlertmanager) sanitizeLabelName(name string) (string, error) { + if len(name) == 0 { + return "", errors.New("label name cannot be empty") + } + + if isValidLabelName(name) { + return name, nil + } + + s.logger.Warn("alert sending to external Alertmanager(s) contains label/annotation name with invalid characters", "name", name) + + // Remove spaces. We do this instead of replacing with underscore for backwards compatibility as this existed before the rest of this function. + sanitized := strings.Join(strings.Fields(name), "") + + // Replace other invalid characters. + var buf strings.Builder + for i, b := range sanitized { + if isValidCharacter(i, b) { + buf.WriteRune(b) + continue + } + + if b <= unicode.MaxASCII { + buf.WriteRune('_') + continue + } + + if i == 0 { + buf.WriteRune('_') + } + _, _ = fmt.Fprintf(&buf, "%#x", b) + } + + if buf.Len() == 0 { + return "", fmt.Errorf("label name is empty after removing invalids chars") + } + + return buf.String(), nil +} + +// isValidLabelName is true iff the label name matches the pattern of ^[a-zA-Z_][a-zA-Z0-9_]*$. +func isValidLabelName(ln string) bool { + if len(ln) == 0 { + return false + } + for i, b := range ln { + if !isValidCharacter(i, b) { + return false + } + } + return true +} + +// isValidCharacter checks if a specific rune is allowed at the given position in a label key for an external Prometheus alertmanager. +// From alertmanager LabelName.IsValid(). +func isValidCharacter(pos int, b rune) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && pos > 0) +} + +func sortedKeys(m map[string]string) []string { + orderedKeys := make([]string, len(m)) + i := 0 + for k := range m { + orderedKeys[i] = k + i++ + } + sort.Strings(orderedKeys) + return orderedKeys } diff --git a/pkg/services/ngalert/sender/sender_test.go b/pkg/services/ngalert/sender/sender_test.go new file mode 100644 index 00000000000..3b51a2c6b9b --- /dev/null +++ b/pkg/services/ngalert/sender/sender_test.go @@ -0,0 +1,103 @@ +package sender + +import ( + "testing" + + "github.com/prometheus/alertmanager/api/v2/models" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/stretchr/testify/require" +) + +func TestSanitizeLabelName(t *testing.T) { + cases := []struct { + desc string + labelName string + expectedResult string + expectedErr string + }{ + { + desc: "Remove whitespace", + labelName: " a\tb\nc\vd\re\ff ", + expectedResult: "abcdef", + }, + { + desc: "Replace ASCII with underscore", + labelName: " !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~", + expectedResult: "________________0123456789_______ABCDEFGHIJKLMNOPQRSTUVWXYZ______abcdefghijklmnopqrstuvwxyz____", + }, + { + desc: "Replace non-ASCII unicode with hex", + labelName: "_€_ƒ_„_†_‡_œ_Ÿ_®_º_¼_×_ð_þ_¿_±_四_十_二_🔥", + expectedResult: "_0x20ac_0x192_0x201e_0x2020_0x2021_0x153_0x178_0xae_0xba_0xbc_0xd7_0xf0_0xfe_0xbf_0xb1_0x56db_0x5341_0x4e8c_0x1f525", + }, + { // labels starting with a number are invalid, so we have to make sure we don't sanitize to another invalid label. + desc: "If first character is replaced with hex, prefix with underscore", + labelName: "😍😍😍", + expectedResult: "_0x1f60d0x1f60d0x1f60d", + }, + { + desc: "Empty string should error", + labelName: "", + expectedErr: "label name cannot be empty", + }, + { + desc: "Only whitespace should error", + labelName: " \t\n\v\n\f ", + expectedErr: "label name is empty after removing invalids chars", + }, + } + + for _, tc := range cases { + am, _ := NewExternalAlertmanagerSender() + t.Run(tc.desc, func(t *testing.T) { + res, err := am.sanitizeLabelName(tc.labelName) + + if tc.expectedErr != "" { + require.EqualError(t, err, tc.expectedErr) + } + + require.Equal(t, tc.expectedResult, res) + }) + } +} + +func TestSanitizeLabelSet(t *testing.T) { + cases := []struct { + desc string + labelset models.LabelSet + expectedResult labels.Labels + }{ + { + desc: "Duplicate labels after sanitizations, append short has as suffix to duplicates", + labelset: models.LabelSet{ + "test-alert": "42", + "test_alert": "43", + "test+alert": "44", + }, + expectedResult: labels.Labels{ + labels.Label{Name: "test_alert", Value: "44"}, + labels.Label{Name: "test_alert_ed6237", Value: "42"}, + labels.Label{Name: "test_alert_a67b5e", Value: "43"}, + }, + }, + { + desc: "If sanitize fails for a label, skip it", + labelset: models.LabelSet{ + "test-alert": "42", + " \t\n\v\n\f ": "43", + "test+alert": "44", + }, + expectedResult: labels.Labels{ + labels.Label{Name: "test_alert", Value: "44"}, + labels.Label{Name: "test_alert_ed6237", Value: "42"}, + }, + }, + } + + for _, tc := range cases { + am, _ := NewExternalAlertmanagerSender() + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.expectedResult, am.sanitizeLabelSet(tc.labelset)) + }) + } +}