Make alerting notifcations sync (#6158)

* tech(routines): move the async logic from notification to alerting notifier

* tech(notification): reduce code dupe

* fix(notification): dont touch the response unless its an error

* feat(alerting): make alerting exeuction async but flow sync

* tech(alerting): remove commented code

* tech(alerting): remove unused code

* tech(alerting): fix typo

* tech(alerting): implement Context on EvalContext

* tech(alerting): wait for all alerts to return

* feat(alerting): dont allow alert responses to cancel

* Revert "feat(alerting): dont allow alert responses to cancel"

This reverts commit 324b006c96.

* feat(alerting): give alerts some time to finish before closing down

pkg/services/notifications/mailer.go

@@ -5,8 +5,11 @@
 package notifications
 
 import (
+	"bytes"
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"html/template"
 	"net"
 	"net/mail"
 	"net/smtp"
@@ -15,6 +18,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -185,3 +189,49 @@ func buildAndSend(msg *Message) (int, error) {
 		}
 	}
 }
+
+func buildEmailMessage(cmd *m.SendEmailCommand) (*Message, error) {
+	if !setting.Smtp.Enabled {
+		return nil, errors.New("Grafana mailing/smtp options not configured, contact your Grafana admin")
+	}
+
+	var buffer bytes.Buffer
+	var err error
+	var subjectText interface{}
+
+	data := cmd.Data
+	if data == nil {
+		data = make(map[string]interface{}, 10)
+	}
+
+	setDefaultTemplateData(data, nil)
+	err = mailTemplates.ExecuteTemplate(&buffer, cmd.Template, data)
+	if err != nil {
+		return nil, err
+	}
+
+	subjectData := data["Subject"].(map[string]interface{})
+	subjectText, hasSubject := subjectData["value"]
+
+	if !hasSubject {
+		return nil, errors.New(fmt.Sprintf("Missing subject in Template %s", cmd.Template))
+	}
+
+	subjectTmpl, err := template.New("subject").Parse(subjectText.(string))
+	if err != nil {
+		return nil, err
+	}
+
+	var subjectBuffer bytes.Buffer
+	err = subjectTmpl.ExecuteTemplate(&subjectBuffer, "subject", data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Message{
+		To:      cmd.To,
+		From:    setting.Smtp.FromAddress,
+		Subject: subjectBuffer.String(),
+		Body:    buffer.String(),
+	}, nil
+}

pkg/services/notifications/notifications.go

@@ -1,7 +1,7 @@
 package notifications
 
 import (
-	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"html/template"
@@ -29,7 +29,10 @@ func Init() error {
 	bus.AddHandler("email", validateResetPasswordCode)
 	bus.AddHandler("email", sendEmailCommandHandler)
 
+	bus.AddCtxHandler("email", sendEmailCommandHandlerSync)
+
 	bus.AddHandler("webhook", sendWebhook)
+	bus.AddCtxHandler("webhook", SendWebhookSync)
 
 	bus.AddEventListener(signUpStartedHandler)
 	bus.AddEventListener(signUpCompletedHandler)
@@ -56,6 +59,15 @@ func Init() error {
 	return nil
 }
 
+func SendWebhookSync(ctx context.Context, cmd *m.SendWebhookSync) error {
+	return sendWebRequestSync(ctx, &Webhook{
+		Url:      cmd.Url,
+		User:     cmd.User,
+		Password: cmd.Password,
+		Body:     cmd.Body,
+	})
+}
+
 func sendWebhook(cmd *m.SendWebhook) error {
 	addToWebhookQueue(&Webhook{
 		Url:      cmd.Url,
@@ -72,51 +84,33 @@ func subjectTemplateFunc(obj map[string]interface{}, value string) string {
 	return ""
 }
 
-func sendEmailCommandHandler(cmd *m.SendEmailCommand) error {
-	if !setting.Smtp.Enabled {
-		return errors.New("Grafana mailing/smtp options not configured, contact your Grafana admin")
-	}
-
-	var buffer bytes.Buffer
-	var err error
-	var subjectText interface{}
-
-	data := cmd.Data
-	if data == nil {
-		data = make(map[string]interface{}, 10)
-	}
-
-	setDefaultTemplateData(data, nil)
-	err = mailTemplates.ExecuteTemplate(&buffer, cmd.Template, data)
-	if err != nil {
-		return err
-	}
-
-	subjectData := data["Subject"].(map[string]interface{})
-	subjectText, hasSubject := subjectData["value"]
-
-	if !hasSubject {
-		return errors.New(fmt.Sprintf("Missing subject in Template %s", cmd.Template))
-	}
-
-	subjectTmpl, err := template.New("subject").Parse(subjectText.(string))
-	if err != nil {
-		return err
-	}
-
-	var subjectBuffer bytes.Buffer
-	err = subjectTmpl.ExecuteTemplate(&subjectBuffer, "subject", data)
-	if err != nil {
-		return err
-	}
-
-	addToMailQueue(&Message{
-		To:      cmd.To,
-		From:    setting.Smtp.FromAddress,
-		Subject: subjectBuffer.String(),
-		Body:    buffer.String(),
+func sendEmailCommandHandlerSync(ctx context.Context, cmd *m.SendEmailCommandSync) error {
+	message, err := buildEmailMessage(&m.SendEmailCommand{
+		Data:     cmd.Data,
+		Info:     cmd.Info,
+		Massive:  cmd.Massive,
+		Template: cmd.Template,
+		To:       cmd.To,
 	})
 
+	if err != nil {
+		return err
+	}
+
+	_, err = buildAndSend(message)
+
+	return err
+}
+
+func sendEmailCommandHandler(cmd *m.SendEmailCommand) error {
+	message, err := buildEmailMessage(cmd)
+
+	if err != nil {
+		return err
+	}
+
+	addToMailQueue(message)
+
 	return nil
 }
 

pkg/services/notifications/notifications_test.go

@@ -3,7 +3,6 @@ package notifications
 import (
 	"testing"
 
-	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 	. "github.com/smartystreets/goconvey/convey"
@@ -18,7 +17,7 @@ type testTriggeredAlert struct {
 func TestNotifications(t *testing.T) {
 
 	Convey("Given the notifications service", t, func() {
-		bus.ClearBusHandlers()
+		//bus.ClearBusHandlers()
 
 		setting.StaticRootPath = "../../../public/"
 		setting.Smtp.Enabled = true

pkg/services/notifications/webhook.go

@@ -2,11 +2,14 @@ package notifications
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"time"
 
+	"golang.org/x/net/context/ctxhttp"
+
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -31,7 +34,7 @@ func processWebhookQueue() {
 	for {
 		select {
 		case webhook := <-webhookQueue:
-			err := sendWebRequest(webhook)
+			err := sendWebRequestSync(context.TODO(), webhook)
 
 			if err != nil {
 				webhookLog.Error("Failed to send webrequest ", "error", err)
@@ -40,14 +43,14 @@ func processWebhookQueue() {
 	}
 }
 
-func sendWebRequest(webhook *Webhook) error {
+func sendWebRequestSync(ctx context.Context, webhook *Webhook) error {
 	webhookLog.Debug("Sending webhook", "url", webhook.Url)
 
-	client := http.Client{
+	client := &http.Client{
 		Timeout: time.Duration(10 * time.Second),
 	}
 
-	request, err := http.NewRequest("POST", webhook.Url, bytes.NewReader([]byte(webhook.Body)))
+	request, err := http.NewRequest(http.MethodPost, webhook.Url, bytes.NewReader([]byte(webhook.Body)))
 	if webhook.User != "" && webhook.Password != "" {
 		request.Header.Add("Authorization", util.GetBasicAuthHeader(webhook.User, webhook.Password))
 	}
@@ -56,22 +59,23 @@ func sendWebRequest(webhook *Webhook) error {
 		return err
 	}
 
-	resp, err := client.Do(request)
+	resp, err := ctxhttp.Do(ctx, client, request)
 	if err != nil {
 		return err
 	}
 
-	_, err = ioutil.ReadAll(resp.Body)
+	if resp.StatusCode/100 == 2 {
+		return nil
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("Webhook response code %v", resp.StatusCode)
-	}
-
 	defer resp.Body.Close()
-	return nil
+
+	webhookLog.Debug("Webhook failed", "statuscode", resp.Status, "body", string(body))
+	return fmt.Errorf("Webhook response status %v", resp.Status)
 }
 
 var addToWebhookQueue = func(msg *Webhook) {