feat(annotations): working on alert annotations, #5694

pkg/services/alerting/engine.go

@@ -38,8 +38,8 @@ func (e *Engine) Start() {
 	e.log.Info("Starting Alerting Engine")
 
 	go e.alertingTicker()
-	go e.execDispatch()
-	go e.resultDispatch()
+	go e.execDispatcher()
+	go e.resultDispatcher()
 }
 
 func (e *Engine) Stop() {
@@ -70,7 +70,7 @@ func (e *Engine) alertingTicker() {
 	}
 }
 
-func (e *Engine) execDispatch() {
+func (e *Engine) execDispatcher() {
 	for job := range e.execQueue {
 		e.log.Debug("Starting executing alert rule %s", job.Rule.Name)
 		go e.executeJob(job)
@@ -92,10 +92,10 @@ func (e *Engine) executeJob(job *Job) {
 	e.resultQueue <- context
 }
 
-func (e *Engine) resultDispatch() {
+func (e *Engine) resultDispatcher() {
 	defer func() {
 		if err := recover(); err != nil {
-			e.log.Error("Engine Panic, stopping resultHandler", "error", err, "stack", log.Stack(1))
+			e.log.Error("Panic in resultDispatcher", "error", err, "stack", log.Stack(1))
 		}
 	}()
 

pkg/services/alerting/eval_context.go

@@ -56,6 +56,10 @@ func (c *EvalContext) GetStateText() string {
 	}
 }
 
+func (c *EvalContext) GetNotificationTitle() string {
+	return "[" + c.GetStateText() + "] " + c.Rule.Name
+}
+
 func (c *EvalContext) getDashboardSlug() (string, error) {
 	if c.dashboardSlug != "" {
 		return c.dashboardSlug, nil

pkg/services/alerting/notifiers/email.go

@@ -47,6 +47,7 @@ func (this *EmailNotifier) Notify(context *alerting.EvalContext) {
 
 	cmd := &m.SendEmailCommand{
 		Data: map[string]interface{}{
+			"Title":     context.GetNotificationTitle(),
 			"RuleState": context.Rule.State,
 			"RuleName":  context.Rule.Name,
 			"Severity":  context.Rule.Severity,

pkg/services/alerting/notifiers/slack.go

@@ -39,8 +39,6 @@ type SlackNotifier struct {
 func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
 	this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
 
-	rule := context.Rule
-
 	ruleUrl, err := context.GetRuleUrl()
 	if err != nil {
 		this.log.Error("Failed get rule link", "error", err)
@@ -68,7 +66,7 @@ func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
 				// "author_name": "Bobby Tables",
 				// "author_link": "http://flickr.com/bobby/",
 				// "author_icon": "http://flickr.com/icons/bobby.jpg",
-				"title":      "[" + context.GetStateText() + "] " + rule.Name,
+				"title":      context.GetNotificationTitle(),
 				"title_link": ruleUrl,
 				// "text":       "Optional text that appears within the attachment",
 				"fields":    fields,

pkg/services/alerting/notifiers/webhook.go

@@ -42,7 +42,9 @@ func (this *WebhookNotifier) Notify(context *alerting.EvalContext) {
 	this.log.Info("Sending webhook")
 
 	bodyJSON := simplejson.New()
-	bodyJSON.Set("name", context.Rule.Name)
+	bodyJSON.Set("title", context.GetNotificationTitle())
+	bodyJSON.Set("ruleId", context.Rule.Id)
+	bodyJSON.Set("ruleName", context.Rule.Name)
 	bodyJSON.Set("firing", context.Firing)
 	bodyJSON.Set("severity", context.Rule.Severity)
 

pkg/services/alerting/result_handler.go

@@ -1,13 +1,16 @@
 package alerting
 
 import (
+	"time"
+
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/annotations"
 )
 
 type ResultHandler interface {
-	Handle(result *EvalContext)
+	Handle(ctx *EvalContext)
 }
 
 type DefaultResultHandler struct {
@@ -22,32 +25,47 @@ func NewResultHandler() *DefaultResultHandler {
 	}
 }
 
-func (handler *DefaultResultHandler) Handle(result *EvalContext) {
-	var newState m.AlertStateType
+func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
+	oldState := ctx.Rule.State
 
-	if result.Error != nil {
-		handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
-		newState = m.AlertStatePending
-	} else if result.Firing {
-		newState = m.AlertStateFiring
+	if ctx.Error != nil {
+		handler.log.Error("Alert Rule Result Error", "ruleId", ctx.Rule.Id, "error", ctx.Error)
+		ctx.Rule.State = m.AlertStatePending
+	} else if ctx.Firing {
+		ctx.Rule.State = m.AlertStateFiring
 	} else {
-		newState = m.AlertStateOK
+		ctx.Rule.State = m.AlertStateOK
 	}
 
-	if result.Rule.State != newState {
-		handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
+	if ctx.Rule.State != oldState {
+		handler.log.Info("New state change", "alertId", ctx.Rule.Id, "newState", ctx.Rule.State, "oldState", oldState)
 
 		cmd := &m.SetAlertStateCommand{
-			AlertId: result.Rule.Id,
-			OrgId:   result.Rule.OrgId,
-			State:   newState,
+			AlertId: ctx.Rule.Id,
+			OrgId:   ctx.Rule.OrgId,
+			State:   ctx.Rule.State,
 		}
 
 		if err := bus.Dispatch(cmd); err != nil {
 			handler.log.Error("Failed to save state", "error", err)
 		}
 
-		result.Rule.State = newState
-		handler.notifier.Notify(result)
+		// save annotation
+		item := annotations.Item{
+			OrgId:     ctx.Rule.OrgId,
+			Type:      annotations.AlertType,
+			AlertId:   ctx.Rule.Id,
+			Title:     ctx.Rule.Name,
+			Text:      ctx.GetStateText(),
+			NewState:  string(ctx.Rule.State),
+			PrevState: string(oldState),
+			Timestamp: time.Now(),
+		}
+		annotationRepo := annotations.GetRepository()
+		if err := annotationRepo.Save(&item); err != nil {
+			handler.log.Error("Failed to save annotation for new alert state", "error", err)
+		}
+
+		handler.notifier.Notify(ctx)
 	}
 }

pkg/services/alerting/scheduler.go

@@ -19,7 +19,7 @@ func NewScheduler() Scheduler {
 }
 
 func (s *SchedulerImpl) Update(rules []*Rule) {
-	s.log.Debug("Scheduling update", "rules.count", len(rules))
+	s.log.Debug("Scheduling update", "ruleCount", len(rules))
 
 	jobs := make(map[int64]*Job, 0)