Make alerting notifcations sync (#6158)

* tech(routines): move the async logic from notification to alerting notifier

* tech(notification): reduce code dupe

* fix(notification): dont touch the response unless its an error

* feat(alerting): make alerting exeuction async but flow sync

* tech(alerting): remove commented code

* tech(alerting): remove unused code

* tech(alerting): fix typo

* tech(alerting): implement Context on EvalContext

* tech(alerting): wait for all alerts to return

* feat(alerting): dont allow alert responses to cancel

* Revert "feat(alerting): dont allow alert responses to cancel"

This reverts commit 324b006c96.

* feat(alerting): give alerts some time to finish before closing down

pkg/services/alerting/eval_handler.go

@@ -1,17 +1,12 @@
 package alerting
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/metrics"
 )
 
-var (
-	MaxRetries int = 1
-)
-
 type DefaultEvalHandler struct {
 	log             log.Logger
 	alertJobTimeout time.Duration
@@ -20,49 +15,11 @@ type DefaultEvalHandler struct {
 func NewEvalHandler() *DefaultEvalHandler {
 	return &DefaultEvalHandler{
 		log:             log.New("alerting.evalHandler"),
-		alertJobTimeout: time.Second * 15,
+		alertJobTimeout: time.Second * 5,
 	}
 }
 
 func (e *DefaultEvalHandler) Eval(context *EvalContext) {
-	go e.eval(context)
-
-	select {
-	case <-time.After(e.alertJobTimeout):
-		context.Error = fmt.Errorf("Execution timed out after %v", e.alertJobTimeout)
-		context.EndTime = time.Now()
-		e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id, "timeout setting", e.alertJobTimeout)
-		e.retry(context)
-	case <-context.DoneChan:
-		e.log.Debug("Job Execution done", "timeMs", context.GetDurationMs(), "alertId", context.Rule.Id, "firing", context.Firing)
-
-		if context.Error != nil {
-			e.retry(context)
-		}
-	}
-}
-
-func (e *DefaultEvalHandler) retry(context *EvalContext) {
-	e.log.Debug("Retrying eval exeuction", "alertId", context.Rule.Id)
-
-	if context.RetryCount < MaxRetries {
-		context.DoneChan = make(chan bool, 1)
-		context.CancelChan = make(chan bool, 1)
-		context.RetryCount++
-		e.Eval(context)
-	}
-}
-
-func (e *DefaultEvalHandler) eval(context *EvalContext) {
-	defer func() {
-		if err := recover(); err != nil {
-			e.log.Error("Alerting rule eval panic", "error", err, "stack", log.Stack(1))
-			if panicErr, ok := err.(error); ok {
-				context.Error = panicErr
-			}
-		}
-	}()
-
 	for _, condition := range context.Rule.Conditions {
 		condition.Eval(context)
 
@@ -80,5 +37,4 @@ func (e *DefaultEvalHandler) eval(context *EvalContext) {
 	context.EndTime = time.Now()
 	elapsedTime := context.EndTime.Sub(context.StartTime) / time.Millisecond
 	metrics.M_Alerting_Exeuction_Time.Update(elapsedTime)
-	context.DoneChan <- true
 }