From 20d94f9703a9c74d5a6d2551b4ddb725d977c5a2 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sat, 17 Jun 2017 12:30:40 +0900 Subject: [PATCH 1/6] support alertmanager --- .../alerting/notifiers/alertmanager.go | 95 +++++++++++++++++++ .../alerting/notifiers/alertmanager_test.go | 50 ++++++++++ 2 files changed, 145 insertions(+) create mode 100644 pkg/services/alerting/notifiers/alertmanager.go create mode 100644 pkg/services/alerting/notifiers/alertmanager_test.go diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go new file mode 100644 index 00000000000..29b2474d8c9 --- /dev/null +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -0,0 +1,95 @@ +package notifiers + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" +) + +func init() { + alerting.RegisterNotifier(&alerting.NotifierPlugin{ + Type: "alertmanager", + Name: "alertmanager", + Description: "Sends alert to Alertmanager", + Factory: NewAlertmanagerNotifier, + OptionsTemplate: ` +

Alertmanager settings

+
+ Url + +
+ `, + }) +} + +func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, error) { + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find url property in settings"} + } + + return &AlertmanagerNotifier{ + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + Url: url, + log: log.New("alerting.notifier.alertmanager"), + }, nil +} + +type AlertmanagerNotifier struct { + NotifierBase + Url string + log log.Logger +} + +func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error { + this.log.Info("Sending alertmanager") + + alerts := make([]interface{}, 0) + for _, match := range evalContext.EvalMatches { + alertJSON := simplejson.New() + alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339)) + if evalContext.Rule.State == m.AlertStateAlerting { + alertJSON.Set("endsAt", "0001-01-01T00:00:00Z") + } else { + alertJSON.Set("endsAt", evalContext.EndTime.UTC().Format(time.RFC3339)) + } + + ruleUrl, err := evalContext.GetRuleUrl() + if err == nil { + alertJSON.Set("generatorURL", ruleUrl) + } + + if evalContext.Rule.Message != "" { + alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message) + } + + tags := make(map[string]string) + for k, v := range match.Tags { + tags[k] = v + } + tags["alertname"] = evalContext.Rule.Name + alertJSON.Set("labels", tags) + + alerts = append(alerts, alertJSON) + } + + bodyJSON := simplejson.NewFromAny(alerts) + body, _ := bodyJSON.MarshalJSON() + + cmd := &m.SendWebhookSync{ + Url: this.Url + "/api/v1/alerts", + HttpMethod: "POST", + Body: string(body), + } + + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { + this.log.Error("Failed to send alertmanager", "error", err, "alertmanager", this.Name) + return err + } + + return nil +} diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go new file mode 100644 index 00000000000..78c9511c783 --- /dev/null +++ b/pkg/services/alerting/notifiers/alertmanager_test.go @@ -0,0 +1,50 @@ +package notifiers + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestAlertmanagerNotifier(t *testing.T) { + Convey("Alertmanager notifier tests", t, func() { + + Convey("Parsing alert notification from settings", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "alertmanager", + Type: "alertmanager", + Settings: settingsJSON, + } + + _, err := NewAlertmanagerNotifier(model) + So(err, ShouldNotBeNil) + }) + + Convey("from settings", func() { + json := ` + { + "url": "http://127.0.0.1:9093/" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "alertmanager", + Type: "alertmanager", + Settings: settingsJSON, + } + + not, err := NewAlertmanagerNotifier(model) + alertmanagerNotifier := not.(*AlertmanagerNotifier) + + So(err, ShouldBeNil) + So(alertmanagerNotifier.Url, ShouldEqual, "http://127.0.0.1:9093/") + }) + }) + }) +} From 3a7939c930052a678e2aaec4e20b4c5e26b7dc74 Mon Sep 17 00:00:00 2001 From: Thibault Chataigner Date: Thu, 16 Nov 2017 10:26:53 +0100 Subject: [PATCH 2/6] Alertmanager notifier: make it match the new notifier interface Signed-off-by: Thibault Chataigner --- pkg/services/alerting/notifiers/alertmanager.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 29b2474d8c9..163cb11b45e 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -45,6 +45,13 @@ type AlertmanagerNotifier struct { log log.Logger } +func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool { + if evalContext.Rule.State == m.AlertStateAlerting { + return true + } + return false +} + func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error { this.log.Info("Sending alertmanager") @@ -52,11 +59,8 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro for _, match := range evalContext.EvalMatches { alertJSON := simplejson.New() alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339)) - if evalContext.Rule.State == m.AlertStateAlerting { - alertJSON.Set("endsAt", "0001-01-01T00:00:00Z") - } else { - alertJSON.Set("endsAt", evalContext.EndTime.UTC().Format(time.RFC3339)) - } + // Rule state should always be alerting if notifying. + alertJSON.Set("endsAt", "0001-01-01T00:00:00Z") ruleUrl, err := evalContext.GetRuleUrl() if err == nil { From a8264fe316a43156096c98bcebdea2b8715aeb76 Mon Sep 17 00:00:00 2001 From: Thibault Chataigner Date: Wed, 13 Dec 2017 13:37:49 +0100 Subject: [PATCH 3/6] Alertmanager notifier: add "metric" labels if no tags Signed-off-by: Thibault Chataigner --- pkg/services/alerting/notifiers/alertmanager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 163cb11b45e..ccf3cef5d16 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -72,8 +72,12 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro } tags := make(map[string]string) - for k, v := range match.Tags { - tags[k] = v + if len(match.Tags) == 0 { + tags["metric"] = match.Metric + } else { + for k, v := range match.Tags { + tags[k] = v + } } tags["alertname"] = evalContext.Rule.Name alertJSON.Set("labels", tags) From 1240db31d3b2de98f1ccade6b36ce662d5d26f40 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 13 Dec 2017 19:12:05 +0100 Subject: [PATCH 4/6] alerting: reduce log level for notifiers --- pkg/services/alerting/notifier.go | 2 +- pkg/services/alerting/notifiers/base.go | 2 +- pkg/services/alerting/notifiers/dingding_test.go | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 32bece31820..c4c4dc45f50 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -65,7 +65,7 @@ func (n *notificationService) sendNotifications(context *EvalContext, notifiers for _, notifier := range notifiers { not := notifier //avoid updating scope variable in go routine - n.log.Info("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) + n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() g.Go(func() error { return not.Notify(context) }) } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 2308a5943d6..601f8fc24b1 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -15,7 +15,7 @@ type NotifierBase struct { } func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase { - uploadImage := model.Get("uploadImage").MustBool(true) + uploadImage := model.Get("uploadImage").MustBool(false) return NotifierBase{ Id: id, diff --git a/pkg/services/alerting/notifiers/dingding_test.go b/pkg/services/alerting/notifiers/dingding_test.go index b21815f95af..f89bf6382ce 100644 --- a/pkg/services/alerting/notifiers/dingding_test.go +++ b/pkg/services/alerting/notifiers/dingding_test.go @@ -25,10 +25,8 @@ func TestDingDingNotifier(t *testing.T) { }) Convey("settings should trigger incident", func() { - json := ` - { - "url": "https://www.google.com" - }` + json := `{ "url": "https://www.google.com" }` + settingsJSON, _ := simplejson.NewJson([]byte(json)) model := &m.AlertNotification{ Name: "dingding_testing", From 3691faf136ff5d22b8f3cf127d8654d9548617c9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 13 Dec 2017 19:13:23 +0100 Subject: [PATCH 5/6] alertmanager: code style --- pkg/services/alerting/notifiers/alertmanager.go | 17 ++++++----------- .../alerting/notifiers/alertmanager_test.go | 5 +---- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index ccf3cef5d16..23cbfdbcce8 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -12,9 +12,9 @@ import ( func init() { alerting.RegisterNotifier(&alerting.NotifierPlugin{ - Type: "alertmanager", - Name: "alertmanager", - Description: "Sends alert to Alertmanager", + Type: "prometheus-alertmanager", + Name: "Prometheus Alertmanager", + Description: "Sends alert to Prometheus Alertmanager", Factory: NewAlertmanagerNotifier, OptionsTemplate: `

Alertmanager settings

@@ -35,7 +35,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err return &AlertmanagerNotifier{ NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), Url: url, - log: log.New("alerting.notifier.alertmanager"), + log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil } @@ -46,14 +46,10 @@ type AlertmanagerNotifier struct { } func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool { - if evalContext.Rule.State == m.AlertStateAlerting { - return true - } - return false + return evalContext.Rule.State == m.AlertStateAlerting } func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error { - this.log.Info("Sending alertmanager") alerts := make([]interface{}, 0) for _, match := range evalContext.EvalMatches { @@ -62,8 +58,7 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro // Rule state should always be alerting if notifying. alertJSON.Set("endsAt", "0001-01-01T00:00:00Z") - ruleUrl, err := evalContext.GetRuleUrl() - if err == nil { + if ruleUrl, err := evalContext.GetRuleUrl(); err == nil { alertJSON.Set("generatorURL", ruleUrl) } diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go index 78c9511c783..3549b536e48 100644 --- a/pkg/services/alerting/notifiers/alertmanager_test.go +++ b/pkg/services/alerting/notifiers/alertmanager_test.go @@ -27,10 +27,7 @@ func TestAlertmanagerNotifier(t *testing.T) { }) Convey("from settings", func() { - json := ` - { - "url": "http://127.0.0.1:9093/" - }` + json := `{ "url": "http://127.0.0.1:9093/" }` settingsJSON, _ := simplejson.NewJson([]byte(json)) model := &m.AlertNotification{ From ade734168cfe273cfe976e36dfe06217fe71cdbe Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 13 Dec 2017 19:15:17 +0100 Subject: [PATCH 6/6] alertmanager: endAt should only be used if we have the correct value --- pkg/services/alerting/notifiers/alertmanager.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 23cbfdbcce8..08f8e8be29c 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -55,8 +55,6 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro for _, match := range evalContext.EvalMatches { alertJSON := simplejson.New() alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339)) - // Rule state should always be alerting if notifying. - alertJSON.Set("endsAt", "0001-01-01T00:00:00Z") if ruleUrl, err := evalContext.GetRuleUrl(); err == nil { alertJSON.Set("generatorURL", ruleUrl)