Make alerting notifcations sync (#6158)

* tech(routines): move the async logic from notification to alerting notifier

* tech(notification): reduce code dupe

* fix(notification): dont touch the response unless its an error

* feat(alerting): make alerting exeuction async but flow sync

* tech(alerting): remove commented code

* tech(alerting): remove unused code

* tech(alerting): fix typo

* tech(alerting): implement Context on EvalContext

* tech(alerting): wait for all alerts to return

* feat(alerting): dont allow alert responses to cancel

* Revert "feat(alerting): dont allow alert responses to cancel"

This reverts commit 324b006c96.

* feat(alerting): give alerts some time to finish before closing down

pkg/services/notifications/webhook.go

@@ -2,11 +2,14 @@ package notifications
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"time"
 
+	"golang.org/x/net/context/ctxhttp"
+
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -31,7 +34,7 @@ func processWebhookQueue() {
 	for {
 		select {
 		case webhook := <-webhookQueue:
-			err := sendWebRequest(webhook)
+			err := sendWebRequestSync(context.TODO(), webhook)
 
 			if err != nil {
 				webhookLog.Error("Failed to send webrequest ", "error", err)
@@ -40,14 +43,14 @@ func processWebhookQueue() {
 	}
 }
 
-func sendWebRequest(webhook *Webhook) error {
+func sendWebRequestSync(ctx context.Context, webhook *Webhook) error {
 	webhookLog.Debug("Sending webhook", "url", webhook.Url)
 
-	client := http.Client{
+	client := &http.Client{
 		Timeout: time.Duration(10 * time.Second),
 	}
 
-	request, err := http.NewRequest("POST", webhook.Url, bytes.NewReader([]byte(webhook.Body)))
+	request, err := http.NewRequest(http.MethodPost, webhook.Url, bytes.NewReader([]byte(webhook.Body)))
 	if webhook.User != "" && webhook.Password != "" {
 		request.Header.Add("Authorization", util.GetBasicAuthHeader(webhook.User, webhook.Password))
 	}
@@ -56,22 +59,23 @@ func sendWebRequest(webhook *Webhook) error {
 		return err
 	}
 
-	resp, err := client.Do(request)
+	resp, err := ctxhttp.Do(ctx, client, request)
 	if err != nil {
 		return err
 	}
 
-	_, err = ioutil.ReadAll(resp.Body)
+	if resp.StatusCode/100 == 2 {
+		return nil
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("Webhook response code %v", resp.StatusCode)
-	}
-
 	defer resp.Body.Close()
-	return nil
+
+	webhookLog.Debug("Webhook failed", "statuscode", resp.Status, "body", string(body))
+	return fmt.Errorf("Webhook response status %v", resp.Status)
 }
 
 var addToWebhookQueue = func(msg *Webhook) {