From 777c1024c9d99658e9a025087074242efba87822 Mon Sep 17 00:00:00 2001
From: Corentin Chary <c.chary@criteo.com>
Date: Thu, 15 Mar 2018 10:07:22 +0100
Subject: [PATCH 1/4] alertmanager: if there are no alerts to send, do nothing

---
 pkg/services/alerting/notifiers/alertmanager.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index 08f8e8be29c..607a97d977c 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -78,6 +78,10 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro
 		alerts = append(alerts, alertJSON)
 	}
 
+	if len(alerts) == 0 {
+		return nil
+	}
+
 	bodyJSON := simplejson.NewFromAny(alerts)
 	body, _ := bodyJSON.MarshalJSON()
 

From d4d2b6d7dae1d3aba5a98d9fb8f3a575ad850a78 Mon Sep 17 00:00:00 2001
From: Corentin Chary <c.chary@criteo.com>
Date: Thu, 15 Mar 2018 10:22:51 +0100
Subject: [PATCH 2/4] notitfiers: avoid ShouldNotify duplication

---
 pkg/services/alerting/notifiers/base.go      | 6 ++++++
 pkg/services/alerting/notifiers/dingding.go  | 4 ----
 pkg/services/alerting/notifiers/email.go     | 4 ----
 pkg/services/alerting/notifiers/hipchat.go   | 4 ----
 pkg/services/alerting/notifiers/kafka.go     | 4 ----
 pkg/services/alerting/notifiers/line.go      | 4 ----
 pkg/services/alerting/notifiers/opsgenie.go  | 4 ----
 pkg/services/alerting/notifiers/pagerduty.go | 4 ----
 pkg/services/alerting/notifiers/pushover.go  | 4 ----
 pkg/services/alerting/notifiers/sensu.go     | 4 ----
 pkg/services/alerting/notifiers/slack.go     | 4 ----
 pkg/services/alerting/notifiers/teams.go     | 4 ----
 pkg/services/alerting/notifiers/telegram.go  | 5 +----
 pkg/services/alerting/notifiers/threema.go   | 4 ----
 pkg/services/alerting/notifiers/victorops.go | 4 ----
 pkg/services/alerting/notifiers/webhook.go   | 4 ----
 16 files changed, 7 insertions(+), 60 deletions(-)

diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 601f8fc24b1..7a3cc71c4db 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -27,15 +27,21 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
 }
 
 func defaultShouldNotify(context *alerting.EvalContext) bool {
+	// Only notify on state change.
 	if context.PrevAlertState == context.Rule.State {
 		return false
 	}
+	// Do not notify when we become OK for the first time.
 	if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) {
 		return false
 	}
 	return true
 }
 
+func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
+	return defaultShouldNotify(context)
+}
+
 func (n *NotifierBase) GetType() string {
 	return n.Type
 }
diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go
index c2029b1173c..e32b9d34f91 100644
--- a/pkg/services/alerting/notifiers/dingding.go
+++ b/pkg/services/alerting/notifiers/dingding.go
@@ -38,10 +38,6 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}, nil
 }
 
-func (this *DingDingNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 type DingDingNotifier struct {
 	NotifierBase
 	Url string
diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go
index 095f7c7156a..562ffbe1269 100644
--- a/pkg/services/alerting/notifiers/email.go
+++ b/pkg/services/alerting/notifiers/email.go
@@ -58,10 +58,6 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}, nil
 }
 
-func (this *EmailNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *EmailNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending alert notification to", "addresses", this.Addresses)
 
diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go
index b65f25b1422..f1f63d42a04 100644
--- a/pkg/services/alerting/notifiers/hipchat.go
+++ b/pkg/services/alerting/notifiers/hipchat.go
@@ -75,10 +75,6 @@ type HipChatNotifier struct {
 	log    log.Logger
 }
 
-func (this *HipChatNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *HipChatNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing hipchat notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 
diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go
index e885d44405d..92f6489106b 100644
--- a/pkg/services/alerting/notifiers/kafka.go
+++ b/pkg/services/alerting/notifiers/kafka.go
@@ -57,10 +57,6 @@ type KafkaNotifier struct {
 	log      log.Logger
 }
 
-func (this *KafkaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *KafkaNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	state := evalContext.Rule.State
diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go
index bc0b0c984a4..4fbaa2d543e 100644
--- a/pkg/services/alerting/notifiers/line.go
+++ b/pkg/services/alerting/notifiers/line.go
@@ -51,10 +51,6 @@ type LineNotifier struct {
 	log   log.Logger
 }
 
-func (this *LineNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *LineNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing line notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 
diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go
index 863b4f1c286..5d8b15160c4 100644
--- a/pkg/services/alerting/notifiers/opsgenie.go
+++ b/pkg/services/alerting/notifiers/opsgenie.go
@@ -72,10 +72,6 @@ type OpsGenieNotifier struct {
 	log       log.Logger
 }
 
-func (this *OpsGenieNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *OpsGenieNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	var err error
diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go
index 6013648d9dd..58484051432 100644
--- a/pkg/services/alerting/notifiers/pagerduty.go
+++ b/pkg/services/alerting/notifiers/pagerduty.go
@@ -65,10 +65,6 @@ type PagerdutyNotifier struct {
 	log         log.Logger
 }
 
-func (this *PagerdutyNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	if evalContext.Rule.State == m.AlertStateOK && !this.AutoResolve {
diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go
index 067c02a4a5d..cbe9e16801a 100644
--- a/pkg/services/alerting/notifiers/pushover.go
+++ b/pkg/services/alerting/notifiers/pushover.go
@@ -123,10 +123,6 @@ type PushoverNotifier struct {
 	log      log.Logger
 }
 
-func (this *PushoverNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *PushoverNotifier) Notify(evalContext *alerting.EvalContext) error {
 	ruleUrl, err := evalContext.GetRuleUrl()
 	if err != nil {
diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go
index 7a34d51b493..9f77801d458 100644
--- a/pkg/services/alerting/notifiers/sensu.go
+++ b/pkg/services/alerting/notifiers/sensu.go
@@ -71,10 +71,6 @@ type SensuNotifier struct {
 	log      log.Logger
 }
 
-func (this *SensuNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *SensuNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending sensu result")
 
diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go
index c5bb9344e30..e051a71740a 100644
--- a/pkg/services/alerting/notifiers/slack.go
+++ b/pkg/services/alerting/notifiers/slack.go
@@ -98,10 +98,6 @@ type SlackNotifier struct {
 	log       log.Logger
 }
 
-func (this *SlackNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing slack notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 
diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go
index 851e5a01c75..9a9e93dbc47 100644
--- a/pkg/services/alerting/notifiers/teams.go
+++ b/pkg/services/alerting/notifiers/teams.go
@@ -47,10 +47,6 @@ type TeamsNotifier struct {
 	log       log.Logger
 }
 
-func (this *TeamsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing teams notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 
diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go
index 88100afe7a1..5cbdad60906 100644
--- a/pkg/services/alerting/notifiers/telegram.go
+++ b/pkg/services/alerting/notifiers/telegram.go
@@ -208,6 +208,7 @@ func generateImageCaption(evalContext *alerting.EvalContext, ruleUrl string, met
 
 	return message
 }
+
 func appendIfPossible(message string, extra string, sizeLimit int) string {
 	if len(extra)+len(message) <= sizeLimit {
 		return message + extra
@@ -216,10 +217,6 @@ func appendIfPossible(message string, extra string, sizeLimit int) string {
 	return message
 }
 
-func (this *TelegramNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error {
 	var cmd *m.SendWebhookSync
 	if evalContext.ImagePublicUrl == "" && this.UploadImage == true {
diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go
index b8455dcfbfd..e4ffffc9108 100644
--- a/pkg/services/alerting/notifiers/threema.go
+++ b/pkg/services/alerting/notifiers/threema.go
@@ -114,10 +114,6 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}, nil
 }
 
-func (this *ThreemaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (notifier *ThreemaNotifier) Notify(evalContext *alerting.EvalContext) error {
 	notifier.log.Info("Sending alert notification from", "threema_id", notifier.GatewayID)
 	notifier.log.Info("Sending alert notification to", "threema_id", notifier.RecipientID)
diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go
index a2b770dfd02..4b4db553cde 100644
--- a/pkg/services/alerting/notifiers/victorops.go
+++ b/pkg/services/alerting/notifiers/victorops.go
@@ -68,10 +68,6 @@ type VictoropsNotifier struct {
 	log         log.Logger
 }
 
-func (this *VictoropsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 // Notify sends notification to Victorops via POST to URL endpoint
 func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go
index d2d6ec636b7..4c97ed2b75e 100644
--- a/pkg/services/alerting/notifiers/webhook.go
+++ b/pkg/services/alerting/notifiers/webhook.go
@@ -65,10 +65,6 @@ type WebhookNotifier struct {
 	log        log.Logger
 }
 
-func (this *WebhookNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *WebhookNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending webhook")
 

From c8da1a46c52207b5292d83812f29b55ddc1cc4eb Mon Sep 17 00:00:00 2001
From: Corentin Chary <c.chary@criteo.com>
Date: Thu, 15 Mar 2018 11:54:07 +0100
Subject: [PATCH 3/4] alertmanager: handle resolved alerts, nodata, and
 execution errors

- Resolved alerts will be sent to the AM the first time they move
  from Alerting to OK
- Errors are forwarded
- NoData is not ignored anymore
- The summary annotation is set (with the alertname)
- The image annotation is set (when available)
---
 .../alerting/notifiers/alertmanager.go        | 76 ++++++++++++++-----
 1 file changed, 59 insertions(+), 17 deletions(-)

diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index 607a97d977c..6f3ee4bf64f 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -46,25 +46,49 @@ type AlertmanagerNotifier struct {
 }
 
 func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
+	this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState)
+
+	// Do not notify when we become OK for the first time.
+	if (evalContext.PrevAlertState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
+		return false
+	}
+	// Notify on Alerting -> OK to resolve before alertmanager timeout.
+	if (evalContext.PrevAlertState == m.AlertStateAlerting) && (evalContext.Rule.State == m.AlertStateOK) {
+		return true
+	}
 	return evalContext.Rule.State == m.AlertStateAlerting
 }
 
-func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
+func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, match *alerting.EvalMatch, ruleUrl string) *simplejson.Json {
+	alertJSON := simplejson.New()
+	alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
+	if evalContext.Rule.State == m.AlertStateOK {
+		alertJSON.Set("endsAt", time.Now().UTC().Format(time.RFC3339))
+	}
+	alertJSON.Set("generatorURL", ruleUrl)
 
-	alerts := make([]interface{}, 0)
-	for _, match := range evalContext.EvalMatches {
-		alertJSON := simplejson.New()
-		alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
-
-		if ruleUrl, err := evalContext.GetRuleUrl(); err == nil {
-			alertJSON.Set("generatorURL", ruleUrl)
+	// Annotations (summary and description are very commonly used).
+	alertJSON.SetPath([]string{"annotations", "summary"}, evalContext.Rule.Name)
+	description := ""
+	if evalContext.Rule.Message != "" {
+		description += evalContext.Rule.Message
+	}
+	if evalContext.Error != nil {
+		if description != "" {
+			description += "\n"
 		}
+		description += "Error: " + evalContext.Error.Error()
+	}
+	if description != "" {
+		alertJSON.SetPath([]string{"annotations", "description"}, description)
+	}
+	if evalContext.ImagePublicUrl != "" {
+		alertJSON.SetPath([]string{"annotations", "image"}, evalContext.ImagePublicUrl)
+	}
 
-		if evalContext.Rule.Message != "" {
-			alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message)
-		}
-
-		tags := make(map[string]string)
+	// Labels (from metrics tags + mandatory alertname).
+	tags := make(map[string]string)
+	if match != nil {
 		if len(match.Tags) == 0 {
 			tags["metric"] = match.Metric
 		} else {
@@ -72,14 +96,32 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro
 				tags[k] = v
 			}
 		}
-		tags["alertname"] = evalContext.Rule.Name
-		alertJSON.Set("labels", tags)
+	}
+	tags["alertname"] = evalContext.Rule.Name
+	alertJSON.Set("labels", tags)
+	return alertJSON
+}
 
-		alerts = append(alerts, alertJSON)
+func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
+	this.log.Info("Creating Altermanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
+
+	ruleUrl, err := evalContext.GetRuleUrl()
+	if err != nil {
+		this.log.Error("Failed get rule link", "error", err)
+		return err
 	}
 
+	// Send one alert per matching series.
+	alerts := make([]interface{}, 0)
+	for _, match := range evalContext.EvalMatches {
+		alert := this.createAlert(evalContext, match, ruleUrl)
+		alerts = append(alerts, alert)
+	}
+
+	// This happens on ExecutionError or NoData
 	if len(alerts) == 0 {
-		return nil
+		alert := this.createAlert(evalContext, nil, ruleUrl)
+		alerts = append(alerts, alert)
 	}
 
 	bodyJSON := simplejson.NewFromAny(alerts)

From 3d8f1a743950430e882f09ea765dc5a7b8508680 Mon Sep 17 00:00:00 2001
From: Corentin Chary <c.chary@criteo.com>
Date: Thu, 15 Mar 2018 13:48:58 +0100
Subject: [PATCH 4/4] alertmanager: /Creating/Sending/

---
 pkg/services/alerting/notifiers/alertmanager.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index 6f3ee4bf64f..d449167de13 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -103,7 +103,7 @@ func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext,
 }
 
 func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
-	this.log.Info("Creating Altermanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
+	this.log.Info("Sending Alertmanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 
 	ruleUrl, err := evalContext.GetRuleUrl()
 	if err != nil {