public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pkg/services: Check errors (#19712)

Browse Source 
* pkg/services: Check errors
* pkg/services: Don't treat context.Canceled|context.DeadlineExceeded as error
This commit is contained in:
Arve Knudsen
2019-10-22 14:08:18 +02:00
committed by GitHub
parent de5038441f
commit 2a78d2a61c
37 changed files with 431 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/conditions/query_test.go
+2 -1
View File
@@ -23,7 +23,8 @@ func TestQueryCondition(t *testing.T) {
ctx.evaluator = `{"type": "gt", "params": [100]}`
Convey("Can read query condition from json model", func() {
ctx.exec()
_, err := ctx.exec()
So(err, ShouldBeNil)
So(ctx.condition.Query.From, ShouldEqual, "5m")
So(ctx.condition.Query.To, ShouldEqual, "now")
pkg/services/alerting/engine.go
+11 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/setting"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
)
// AlertEngine is the background process that
@@ -210,7 +211,16 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
// dont reuse the evalContext and get its own context.
evalContext.Ctx = resultHandleCtx
evalContext.Rule.State = evalContext.GetNewState()
e.resultHandler.handle(evalContext)
if err := e.resultHandler.handle(evalContext); err != nil {
if xerrors.Is(err, context.Canceled) {
e.log.Debug("Result handler returned context.Canceled")
} else if xerrors.Is(err, context.DeadlineExceeded) {
e.log.Debug("Result handler returned context.DeadlineExceeded")
} else {
e.log.Error("Failed to handle result", "err", err)
}
}
span.Finish()
e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
close(attemptChan)
pkg/services/alerting/engine_integration_test.go
+4 -2
View File
@@ -18,7 +18,8 @@ import (
func TestEngineTimeouts(t *testing.T) {
Convey("Alerting engine timeout tests", t, func() {
engine := &AlertEngine{}
engine.Init()
err := engine.Init()
So(err, ShouldBeNil)
setting.AlertingNotificationTimeout = 30 * time.Second
setting.AlertingMaxAttempts = 3
engine.resultHandler = &FakeResultHandler{}
@@ -36,7 +37,8 @@ func TestEngineTimeouts(t *testing.T) {
engine.evalHandler = evalHandler
engine.resultHandler = resultHandler
engine.processJobWithRetry(context.TODO(), job)
err := engine.processJobWithRetry(context.TODO(), job)
So(err, ShouldBeNil)
So(evalHandler.EvalSucceed, ShouldEqual, true)
So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
pkg/services/alerting/engine_test.go
+8 -4
View File
@@ -40,7 +40,8 @@ func (handler *FakeResultHandler) handle(evalContext *EvalContext) error {
func TestEngineProcessJob(t *testing.T) {
Convey("Alerting engine job processing", t, func() {
engine := &AlertEngine{}
engine.Init()
err := engine.Init()
So(err, ShouldBeNil)
setting.AlertingEvaluationTimeout = 30 * time.Second
setting.AlertingNotificationTimeout = 30 * time.Second
setting.AlertingMaxAttempts = 3
@@ -99,7 +100,8 @@ func TestEngineProcessJob(t *testing.T) {
evalHandler := NewFakeEvalHandler(0)
engine.evalHandler = evalHandler
engine.processJobWithRetry(context.TODO(), job)
err := engine.processJobWithRetry(context.TODO(), job)
So(err, ShouldBeNil)
So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
})
@@ -108,7 +110,8 @@ func TestEngineProcessJob(t *testing.T) {
evalHandler := NewFakeEvalHandler(1)
engine.evalHandler = evalHandler
engine.processJobWithRetry(context.TODO(), job)
err := engine.processJobWithRetry(context.TODO(), job)
So(err, ShouldBeNil)
So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
})
@@ -117,7 +120,8 @@ func TestEngineProcessJob(t *testing.T) {
evalHandler := NewFakeEvalHandler(expectedAttempts)
engine.evalHandler = evalHandler
engine.processJobWithRetry(context.TODO(), job)
err := engine.processJobWithRetry(context.TODO(), job)
So(err, ShouldBeNil)
So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
})
})
pkg/services/alerting/notifiers/telegram.go
+46 -19
View File
@@ -89,19 +89,20 @@ func NewTelegramNotifier(model *models.AlertNotification) (alerting.Notifier, er
}, nil
}
func (tn *TelegramNotifier) buildMessage(evalContext *alerting.EvalContext, sendImageInline bool) *models.SendWebhookSync {
func (tn *TelegramNotifier) buildMessage(evalContext *alerting.EvalContext, sendImageInline bool) (*models.SendWebhookSync, error) {
if sendImageInline {
cmd, err := tn.buildMessageInlineImage(evalContext)
if err == nil {
return cmd
return cmd, nil
}
tn.log.Error("Could not generate Telegram message with inline image.", "err", err)
}
return tn.buildMessageLinkedImage(evalContext)
}
func (tn *TelegramNotifier) buildMessageLinkedImage(evalContext *alerting.EvalContext) *models.SendWebhookSync {
func (tn *TelegramNotifier) buildMessageLinkedImage(evalContext *alerting.EvalContext) (*models.SendWebhookSync, error) {
message := fmt.Sprintf("<b>%s</b>\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message)
ruleURL, err := evalContext.GetRuleURL()
@@ -118,11 +119,17 @@ func (tn *TelegramNotifier) buildMessageLinkedImage(evalContext *alerting.EvalCo
message = message + fmt.Sprintf("\n<i>Metrics:</i>%s", metrics)
}
cmd := tn.generateTelegramCmd(message, "text", "sendMessage", func(w *multipart.Writer) {
fw, _ := w.CreateFormField("parse_mode")
fw.Write([]byte("html"))
return tn.generateTelegramCmd(message, "text", "sendMessage", func(w *multipart.Writer) {
fw, err := w.CreateFormField("parse_mode")
if err != nil {
tn.log.Error("Failed to create form file", "err", err)
return
}
if _, err := fw.Write([]byte("html")); err != nil {
tn.log.Error("Failed to write to form field", "err", err)
}
})
return cmd
}
func (tn *TelegramNotifier) buildMessageInlineImage(evalContext *alerting.EvalContext) (*models.SendWebhookSync, error) {
@@ -149,22 +156,38 @@ func (tn *TelegramNotifier) buildMessageInlineImage(evalContext *alerting.EvalCo
metrics := generateMetricsMessage(evalContext)
message := generateImageCaption(evalContext, ruleURL, metrics)
cmd := tn.generateTelegramCmd(message, "caption", "sendPhoto", func(w *multipart.Writer) {
fw, _ := w.CreateFormFile("photo", evalContext.ImageOnDiskPath)
io.Copy(fw, imageFile)
return tn.generateTelegramCmd(message, "caption", "sendPhoto", func(w *multipart.Writer) {
fw, err := w.CreateFormFile("photo", evalContext.ImageOnDiskPath)
if err != nil {
tn.log.Error("Failed to create form file", "err", err)
return
}
if _, err := io.Copy(fw, imageFile); err != nil {
tn.log.Error("Failed to write to form file", "err", err)
}
})
return cmd, nil
}
func (tn *TelegramNotifier) generateTelegramCmd(message string, messageField string, apiAction string, extraConf func(writer *multipart.Writer)) *models.SendWebhookSync {
func (tn *TelegramNotifier) generateTelegramCmd(message string, messageField string, apiAction string, extraConf func(writer *multipart.Writer)) (*models.SendWebhookSync, error) {
var body bytes.Buffer
w := multipart.NewWriter(&body)
fw, _ := w.CreateFormField("chat_id")
fw.Write([]byte(tn.ChatID))
fw, err := w.CreateFormField("chat_id")
if err != nil {
return nil, err
}
if _, err := fw.Write([]byte(tn.ChatID)); err != nil {
return nil, err
}
fw, _ = w.CreateFormField(messageField)
fw.Write([]byte(message))
fw, err = w.CreateFormField(messageField)
if err != nil {
return nil, err
}
if _, err := fw.Write([]byte(message)); err != nil {
return nil, err
}
extraConf(w)
@@ -181,7 +204,7 @@ func (tn *TelegramNotifier) generateTelegramCmd(message string, messageField str
"Content-Type": w.FormDataContentType(),
},
}
return cmd
return cmd, nil
}
func generateMetricsMessage(evalContext *alerting.EvalContext) string {
@@ -232,10 +255,14 @@ func appendIfPossible(message string, extra string, sizeLimit int) string {
// Notify send an alert notification to Telegram.
func (tn *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error {
var cmd *models.SendWebhookSync
var err error
if evalContext.ImagePublicURL == "" && tn.UploadImage {
cmd = tn.buildMessage(evalContext, true)
cmd, err = tn.buildMessage(evalContext, true)
} else {
cmd = tn.buildMessage(evalContext, false)
cmd, err = tn.buildMessage(evalContext, false)
}
if err != nil {
return err
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
pkg/services/alerting/result_handler.go
+12 -1
View File
@@ -1,6 +1,7 @@
package alerting
import (
"context"
"time"
"github.com/grafana/grafana/pkg/bus"
@@ -8,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/models"
"golang.org/x/xerrors"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/rendering"
@@ -98,6 +100,15 @@ func (handler *defaultResultHandler) handle(evalContext *EvalContext) error {
}
}
handler.notifier.SendIfNeeded(evalContext)
if err := handler.notifier.SendIfNeeded(evalContext); err != nil {
if xerrors.Is(err, context.Canceled) {
handler.log.Debug("handler.notifier.SendIfNeeded returned context.Canceled")
} else if xerrors.Is(err, context.DeadlineExceeded) {
handler.log.Debug("handler.notifier.SendIfNeeded returned context.DeadlineExceeded")
} else {
handler.log.Error("handler.notifier.SendIfNeeded failed", "err", err)
}
}
return nil
}