feat(alerting): add support for retries

ref #5855
2016-09-06 15:09:49 +02:00
parent 0d7a871223
commit 04690ca920
2 changed files with 22 additions and 0 deletions
@@ -8,6 +8,10 @@ import (
 	"github.com/grafana/grafana/pkg/metrics"
 )

+var (
+	MaxRetries int = 1
+)
+
 type DefaultEvalHandler struct {
 	log             log.Logger
 	alertJobTimeout time.Duration
@@ -28,8 +32,24 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) {
 		context.Error = fmt.Errorf("Timeout")
 		context.EndTime = time.Now()
 		e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
+		e.retry(context)
 	case <-context.DoneChan:
 		e.log.Debug("Job Execution done", "timeMs", context.GetDurationMs(), "alertId", context.Rule.Id, "firing", context.Firing)
+
+		if context.Error != nil {
+			e.retry(context)
+		}
+	}
+}
+
+func (e *DefaultEvalHandler) retry(context *EvalContext) {
+	e.log.Debug("Retrying eval exeuction", "alertId", context.Rule.Id)
+
+	context.RetryCount++
+	if context.RetryCount > MaxRetries {
+		context.DoneChan = make(chan bool, 1)
+		context.CancelChan = make(chan bool, 1)
+		e.Eval(context)
 	}
 }