From 2904e2d9fe256a55048ecdd6376ef55afed619ba Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Thu, 16 May 2019 11:41:57 +0200 Subject: [PATCH] Alertmanager: Replace illegal chars with underscore in label names (#17002) closes #16624 --- .../alerting/notifiers/alertmanager.go | 45 ++++++++++++------- .../alerting/notifiers/alertmanager_test.go | 27 +++++++++-- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index b90f9e65792..bc2807d0d3c 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -2,6 +2,7 @@ package notifiers import ( "context" + "regexp" "time" "github.com/grafana/grafana/pkg/bus" @@ -27,6 +28,7 @@ func init() { }) } +// NewAlertmanagerNotifier returns a new Alertmanager notifier func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier, error) { url := model.Settings.Get("url").MustString() if url == "" { @@ -35,38 +37,42 @@ func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier return &AlertmanagerNotifier{ NotifierBase: NewNotifierBase(model), - Url: url, + URL: url, log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil } +// AlertmanagerNotifier sends alert notifications to the alert manager type AlertmanagerNotifier struct { NotifierBase - Url string + URL string log log.Logger } -func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext, notificationState *models.AlertNotificationState) bool { - this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState) +// ShouldNotify returns true if the notifiers should be used depending on state +func (am *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext, notificationState *models.AlertNotificationState) bool { + am.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState) // Do not notify when we become OK for the first time. if (evalContext.PrevAlertState == models.AlertStatePending) && (evalContext.Rule.State == models.AlertStateOK) { return false } + // Notify on Alerting -> OK to resolve before alertmanager timeout. if (evalContext.PrevAlertState == models.AlertStateAlerting) && (evalContext.Rule.State == models.AlertStateOK) { return true } + return evalContext.Rule.State == models.AlertStateAlerting } -func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, match *alerting.EvalMatch, ruleUrl string) *simplejson.Json { +func (am *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, match *alerting.EvalMatch, ruleURL string) *simplejson.Json { alertJSON := simplejson.New() alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339)) if evalContext.Rule.State == models.AlertStateOK { alertJSON.Set("endsAt", time.Now().UTC().Format(time.RFC3339)) } - alertJSON.Set("generatorURL", ruleUrl) + alertJSON.Set("generatorURL", ruleURL) // Annotations (summary and description are very commonly used). alertJSON.SetPath([]string{"annotations", "summary"}, evalContext.Rule.Name) @@ -94,7 +100,7 @@ func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, tags["metric"] = match.Metric } else { for k, v := range match.Tags { - tags[k] = v + tags[replaceIllegalCharsInLabelname(k)] = v } } } @@ -103,25 +109,26 @@ func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, return alertJSON } -func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error { - this.log.Info("Sending Alertmanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name) +// Notify sends alert notifications to the alert manager +func (am *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error { + am.log.Info("Sending Alertmanager alert", "ruleId", evalContext.Rule.Id, "notification", am.Name) - ruleUrl, err := evalContext.GetRuleUrl() + ruleURL, err := evalContext.GetRuleUrl() if err != nil { - this.log.Error("Failed get rule link", "error", err) + am.log.Error("Failed get rule link", "error", err) return err } // Send one alert per matching series. alerts := make([]interface{}, 0) for _, match := range evalContext.EvalMatches { - alert := this.createAlert(evalContext, match, ruleUrl) + alert := am.createAlert(evalContext, match, ruleURL) alerts = append(alerts, alert) } // This happens on ExecutionError or NoData if len(alerts) == 0 { - alert := this.createAlert(evalContext, nil, ruleUrl) + alert := am.createAlert(evalContext, nil, ruleURL) alerts = append(alerts, alert) } @@ -129,15 +136,23 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro body, _ := bodyJSON.MarshalJSON() cmd := &models.SendWebhookSync{ - Url: this.Url + "/api/v1/alerts", + Url: am.URL + "/api/v1/alerts", HttpMethod: "POST", Body: string(body), } if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { - this.log.Error("Failed to send alertmanager", "error", err, "alertmanager", this.Name) + am.log.Error("Failed to send alertmanager", "error", err, "alertmanager", am.Name) return err } return nil } + +// regexp that matches all none valid label name characters +// https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels +var labelNamePattern = regexp.MustCompile(`[^a-zA-Z0-9_]`) + +func replaceIllegalCharsInLabelname(input string) string { + return labelNamePattern.ReplaceAllString(input, "_") +} diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go index acf039fbf68..32379caac31 100644 --- a/pkg/services/alerting/notifiers/alertmanager_test.go +++ b/pkg/services/alerting/notifiers/alertmanager_test.go @@ -4,14 +4,35 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/alerting" . "github.com/smartystreets/goconvey/convey" ) +func TestReplaceIllegalCharswithUnderscore(t *testing.T) { + cases := []struct { + input string + expected string + }{ + { + input: "foobar", + expected: "foobar", + }, + { + input: `foo.,\][!?#="~*^&+|<>\'bar09_09`, + expected: "foo____________________bar09_09", + }, + } + + for _, c := range cases { + assert.Equal(t, replaceIllegalCharsInLabelname(c.input), c.expected) + } +} + func TestWhenAlertManagerShouldNotify(t *testing.T) { tcs := []struct { prevState models.AlertStateType @@ -43,7 +64,7 @@ func TestWhenAlertManagerShouldNotify(t *testing.T) { for _, tc := range tcs { am := &AlertmanagerNotifier{log: log.New("test.logger")} - evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ State: tc.prevState, }) @@ -88,7 +109,7 @@ func TestAlertmanagerNotifier(t *testing.T) { alertmanagerNotifier := not.(*AlertmanagerNotifier) So(err, ShouldBeNil) - So(alertmanagerNotifier.Url, ShouldEqual, "http://127.0.0.1:9093/") + So(alertmanagerNotifier.URL, ShouldEqual, "http://127.0.0.1:9093/") }) }) })