public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

alerting: move all notification conditions to defaultShouldNotify

Browse Source
This commit is contained in:
bergquist
2018-09-26 14:34:10 +02:00
parent 8b5aefae26
commit b04052f515
6 changed files with 110 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/notifier.go
+1 -1
View File
@@ -68,7 +68,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
// Verify that we can send the notification again
// but this time within the same transaction.
if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) {
if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) {
return nil
}
pkg/services/alerting/notifiers/base.go
+11 -10
View File
@@ -42,12 +42,21 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase {
}
}
func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify time.Time) bool {
func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, journals []models.AlertNotificationJournal) bool {
// Only notify on state change.
if context.PrevAlertState == context.Rule.State && !sendReminder {
return false
}
// get last successfully sent notification
lastNotify := time.Time{}
for _, j := range journals {
if j.Success {
lastNotify = time.Unix(j.SentAt, 0)
break
}
}
// Do not notify if interval has not elapsed
if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) {
return false
@@ -75,20 +84,12 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext
}
err := bus.DispatchCtx(ctx, cmd)
if err == models.ErrJournalingNotFound {
return true
}
if err != nil {
n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
return false
}
if !cmd.Result.Success {
return true
}
return defaultShouldNotify(c, n.SendReminder, n.Frequency, time.Unix(cmd.Result.SentAt, 0))
return defaultShouldNotify(c, n.SendReminder, n.Frequency, cmd.Result)
}
func (n *NotifierBase) GetType() string {
pkg/services/alerting/notifiers/base_test.go
+73 -29
View File
@@ -15,51 +15,105 @@ import (
)
func TestShouldSendAlertNotification(t *testing.T) {
tnow := time.Now()
tcs := []struct {
name string
prevState m.AlertStateType
newState m.AlertStateType
expected bool
sendReminder bool
frequency time.Duration
journals []m.AlertNotificationJournal
expect bool
}{
{
name: "pending -> ok should not trigger an notification",
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
expected: false,
name: "pending -> ok should not trigger an notification",
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
expect: false,
},
{
name: "ok -> alerting should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
expected: true,
name: "ok -> alerting should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
expect: true,
},
{
name: "ok -> pending should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
expected: false,
name: "ok -> pending should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
expect: false,
},
{
name: "ok -> ok should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
expected: false,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
expect: false,
},
{
name: "ok -> alerting should not trigger an notification",
name: "ok -> alerting should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
expected: true,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
expect: true,
},
{
name: "ok -> ok with reminder should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
expected: false,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
expect: false,
},
{
name: "alerting -> alerting with reminder and no journaling should trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
expect: true,
},
{
name: "alerting -> alerting with reminder and successful recent journal event should not trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
journals: []m.AlertNotificationJournal{
{SentAt: tnow.Add(-time.Minute).Unix(), Success: true},
},
expect: false,
},
{
name: "alerting -> alerting with reminder and failed recent journal event should trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
expect: true,
journals: []m.AlertNotificationJournal{
{SentAt: tnow.Add(-time.Minute).Unix(), Success: false}, // recent failed notification
{SentAt: tnow.Add(-time.Hour).Unix(), Success: true}, // old successful notification
},
},
}
@@ -69,8 +123,8 @@ func TestShouldSendAlertNotification(t *testing.T) {
})
evalContext.Rule.State = tc.prevState
if defaultShouldNotify(evalContext, true, 0, time.Now()) != tc.expected {
t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected)
if defaultShouldNotify(evalContext, true, tc.frequency, tc.journals) != tc.expect {
t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect)
}
}
}
@@ -87,16 +141,6 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) {
})
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{})
Convey("should notify if no journaling is found", func() {
bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error {
return m.ErrJournalingNotFound
})
if !notifier.ShouldNotify(context.Background(), evalContext) {
t.Errorf("should send notifications when ErrJournalingNotFound is returned")
}
})
Convey("should not notify query returns error", func() {
bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error {
return errors.New("some kind of error unknown error")