pkg/services: Check errors (#19712)

* pkg/services: Check errors
* pkg/services: Don't treat context.Canceled|context.DeadlineExceeded as error

pkg/services/alerting/engine.go

@@ -15,6 +15,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/rendering"
 	"github.com/grafana/grafana/pkg/setting"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
 )
 
 // AlertEngine is the background process that
@@ -210,7 +211,16 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 		// dont reuse the evalContext and get its own context.
 		evalContext.Ctx = resultHandleCtx
 		evalContext.Rule.State = evalContext.GetNewState()
-		e.resultHandler.handle(evalContext)
+		if err := e.resultHandler.handle(evalContext); err != nil {
+			if xerrors.Is(err, context.Canceled) {
+				e.log.Debug("Result handler returned context.Canceled")
+			} else if xerrors.Is(err, context.DeadlineExceeded) {
+				e.log.Debug("Result handler returned context.DeadlineExceeded")
+			} else {
+				e.log.Error("Failed to handle result", "err", err)
+			}
+		}
+
 		span.Finish()
 		e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
 		close(attemptChan)