public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13440 from grafana/reminder_refactoring

Browse Source 
Transaction issues for alert reminder
This commit is contained in:
Marcus Efraimsson
2018-10-02 15:55:18 +02:00
committed by GitHub
parent 321c09aec4 0e0901874b
commit 9af809ff8a
23 changed files with 1668 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/interfaces.go
+12 -5
View File
@@ -3,6 +3,8 @@ package alerting
import (
"context"
"time"
"github.com/grafana/grafana/pkg/models"
)
type EvalHandler interface {
@@ -20,7 +22,7 @@ type Notifier interface {
NeedsImage() bool
// ShouldNotify checks this evaluation should send an alert notification
ShouldNotify(ctx context.Context, evalContext *EvalContext) bool
ShouldNotify(ctx context.Context, evalContext *EvalContext, notificationState *models.AlertNotificationState) bool
GetNotifierId() int64
GetIsDefault() bool
@@ -28,11 +30,16 @@ type Notifier interface {
GetFrequency() time.Duration
}
type NotifierSlice []Notifier
type notifierState struct {
notifier Notifier
state *models.AlertNotificationState
}
func (notifiers NotifierSlice) ShouldUploadImage() bool {
for _, notifier := range notifiers {
if notifier.NeedsImage() {
type notifierStateSlice []*notifierState
func (notifiers notifierStateSlice) ShouldUploadImage() bool {
for _, ns := range notifiers {
if ns.notifier.NeedsImage() {
return true
}
}
pkg/services/alerting/notifier.go
+70 -39
View File
@@ -1,10 +1,8 @@
package alerting
import (
"context"
"errors"
"fmt"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/imguploader"
@@ -41,61 +39,78 @@ type notificationService struct {
}
func (n *notificationService) SendIfNeeded(context *EvalContext) error {
notifiers, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context)
notifierStates, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context)
if err != nil {
return err
}
if len(notifiers) == 0 {
if len(notifierStates) == 0 {
return nil
}
if notifiers.ShouldUploadImage() {
if notifierStates.ShouldUploadImage() {
if err = n.uploadImage(context); err != nil {
n.log.Error("Failed to upload alert panel image.", "error", err)
}
}
return n.sendNotifications(context, notifiers)
return n.sendNotifications(context, notifierStates)
}
func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error {
for _, notifier := range notifiers {
not := notifier
func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *notifierState) error {
notifier := notifierState.notifier
err := bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error {
n.log.Debug("trying to send notification", "id", not.GetNotifierId())
n.log.Debug("Sending notification", "type", notifier.GetType(), "id", notifier.GetNotifierId(), "isDefault", notifier.GetIsDefault())
metrics.M_Alerting_Notification_Sent.WithLabelValues(notifier.GetType()).Inc()
// Verify that we can send the notification again
// but this time within the same transaction.
if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) {
return nil
}
err := notifier.Notify(evalContext)
n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
if err != nil {
n.log.Error("failed to send notification", "id", notifier.GetNotifierId(), "error", err)
}
//send notification
success := not.Notify(evalContext) == nil
if evalContext.IsTestRun {
return nil
}
if evalContext.IsTestRun {
return nil
}
cmd := &m.SetAlertNotificationStateToCompleteCommand{
Id: notifierState.state.Id,
Version: notifierState.state.Version,
}
//write result to db.
cmd := &m.RecordNotificationJournalCommand{
OrgId: evalContext.Rule.OrgId,
AlertId: evalContext.Rule.Id,
NotifierId: not.GetNotifierId(),
SentAt: time.Now().Unix(),
Success: success,
}
return bus.DispatchCtx(evalContext.Ctx, cmd)
}
return bus.DispatchCtx(ctx, cmd)
})
func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *notifierState) error {
if !evalContext.IsTestRun {
setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{
Id: notifierState.state.Id,
Version: notifierState.state.Version,
AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges,
}
err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd)
if err == m.ErrAlertNotificationStateVersionConflict {
return nil
}
if err != nil {
n.log.Error("failed to send notification", "id", not.GetNotifierId())
return err
}
// We need to update state version to be able to log
// unexpected version conflicts when marking notifications as ok
notifierState.state.Version = setPendingCmd.ResultVersion
}
return n.sendAndMarkAsComplete(evalContext, notifierState)
}
func (n *notificationService) sendNotifications(evalContext *EvalContext, notifierStates notifierStateSlice) error {
for _, notifierState := range notifierStates {
err := n.sendNotification(evalContext, notifierState)
if err != nil {
n.log.Error("failed to send notification", "id", notifierState.notifier.GetNotifierId(), "error", err)
}
}
@@ -142,22 +157,38 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) {
return nil
}
func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierSlice, error) {
func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (notifierStateSlice, error) {
query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds}
if err := bus.Dispatch(query); err != nil {
return nil, err
}
var result []Notifier
var result notifierStateSlice
for _, notification := range query.Result {
not, err := n.createNotifierFor(notification)
if err != nil {
return nil, err
n.log.Error("Could not create notifier", "notifier", notification.Id, "error", err)
continue
}
if not.ShouldNotify(evalContext.Ctx, evalContext) {
result = append(result, not)
query := &m.GetOrCreateNotificationStateQuery{
NotifierId: notification.Id,
AlertId: evalContext.Rule.Id,
OrgId: evalContext.Rule.OrgId,
}
err = bus.DispatchCtx(evalContext.Ctx, query)
if err != nil {
n.log.Error("Could not get notification state.", "notifier", notification.Id, "error", err)
continue
}
if not.ShouldNotify(evalContext.Ctx, evalContext, query.Result) {
result = append(result, &notifierState{
notifier: not,
state: query.Result,
})
}
}
pkg/services/alerting/notifiers/alertmanager.go
+1 -1
View File
@@ -46,7 +46,7 @@ type AlertmanagerNotifier struct {
log log.Logger
}
func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext) bool {
func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext, notificationState *m.AlertNotificationState) bool {
this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState)
// Do not notify when we become OK for the first time.
pkg/services/alerting/notifiers/base.go
+27 -37
View File
@@ -4,7 +4,6 @@ import (
"context"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
@@ -46,56 +45,47 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase {
}
}
func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, journals []models.AlertNotificationJournal) bool {
// ShouldNotify checks this evaluation should send an alert notification
func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalContext, notiferState *models.AlertNotificationState) bool {
// Only notify on state change.
if context.PrevAlertState == context.Rule.State && !sendReminder {
if context.PrevAlertState == context.Rule.State && !n.SendReminder {
return false
}
// get last successfully sent notification
lastNotify := time.Time{}
for _, j := range journals {
if j.Success {
lastNotify = time.Unix(j.SentAt, 0)
break
if context.PrevAlertState == context.Rule.State && n.SendReminder {
// Do not notify if interval has not elapsed
lastNotify := time.Unix(notiferState.UpdatedAt, 0)
if notiferState.UpdatedAt != 0 && lastNotify.Add(n.Frequency).After(time.Now()) {
return false
}
// Do not notify if alert state is OK or pending even on repeated notify
if context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending {
return false
}
}
// Do not notify if interval has not elapsed
if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) {
return false
}
// Do not notify if alert state if OK or pending even on repeated notify
if sendReminder && (context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending) {
return false
}
// Do not notify when we become OK for the first time.
if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) {
if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
return false
}
// Do not notify when we OK -> Pending
if context.PrevAlertState == models.AlertStateOK && context.Rule.State == models.AlertStatePending {
return false
}
// Do not notifu if state pending and it have been updated last minute
if notiferState.State == models.AlertNotificationStatePending {
lastUpdated := time.Unix(notiferState.UpdatedAt, 0)
if lastUpdated.Add(1 * time.Minute).After(time.Now()) {
return false
}
}
return true
}
// ShouldNotify checks this evaluation should send an alert notification
func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext) bool {
cmd := &models.GetLatestNotificationQuery{
OrgId: c.Rule.OrgId,
AlertId: c.Rule.Id,
NotifierId: n.Id,
}
err := bus.DispatchCtx(ctx, cmd)
if err != nil {
n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
return false
}
return defaultShouldNotify(c, n.SendReminder, n.Frequency, cmd.Result)
}
func (n *NotifierBase) GetType() string {
return n.Type
}
pkg/services/alerting/notifiers/base_test.go
+63 -66
View File
@@ -2,12 +2,9 @@ package notifiers
import (
"context"
"errors"
"testing"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
@@ -23,34 +20,34 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState m.AlertStateType
sendReminder bool
frequency time.Duration
journals []m.AlertNotificationJournal
state *m.AlertNotificationState
expect bool
}{
{
name: "pending -> ok should not trigger an notification",
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
state: &m.AlertNotificationState{},
expect: false,
},
{
name: "ok -> alerting should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
newState: m.AlertStateAlerting,
prevState: m.AlertStateOK,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
state: &m.AlertNotificationState{},
expect: true,
},
{
name: "ok -> pending should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
state: &m.AlertNotificationState{},
expect: false,
},
@@ -59,100 +56,100 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: false,
journals: []m.AlertNotificationJournal{},
state: &m.AlertNotificationState{},
expect: false,
},
{
name: "ok -> alerting should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
expect: true,
},
{
name: "ok -> ok with reminder should not trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
state: &m.AlertNotificationState{},
expect: false,
},
{
name: "alerting -> alerting with reminder and no journaling should trigger",
newState: m.AlertStateAlerting,
name: "alerting -> ok should trigger an notification",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
journals: []m.AlertNotificationJournal{},
sendReminder: false,
state: &m.AlertNotificationState{},
expect: true,
},
{
name: "alerting -> alerting with reminder and successful recent journal event should not trigger",
name: "alerting -> ok should trigger an notification when reminders enabled",
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()},
expect: true,
},
{
name: "alerting -> alerting with reminder and no state should trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
journals: []m.AlertNotificationJournal{
{SentAt: tnow.Add(-time.Minute).Unix(), Success: true},
},
state: &m.AlertNotificationState{},
expect: true,
},
{
name: "alerting -> alerting with reminder and last notification sent 1 minute ago should not trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()},
expect: false,
},
{
name: "alerting -> alerting with reminder and failed recent journal event should trigger",
name: "alerting -> alerting with reminder and last notifciation sent 11 minutes ago should trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
expect: true,
journals: []m.AlertNotificationJournal{
{SentAt: tnow.Add(-time.Minute).Unix(), Success: false}, // recent failed notification
{SentAt: tnow.Add(-time.Hour).Unix(), Success: true}, // old successful notification
},
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-11 * time.Minute).Unix()},
expect: true,
},
{
name: "OK -> alerting with notifciation state pending and updated 30 seconds ago should not trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-30 * time.Second).Unix()},
expect: false,
},
{
name: "OK -> alerting with notifciation state pending and updated 2 minutes ago should trigger",
newState: m.AlertStateAlerting,
prevState: m.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
expect: true,
},
}
for _, tc := range tcs {
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
State: tc.newState,
State: tc.prevState,
})
evalContext.Rule.State = tc.prevState
if defaultShouldNotify(evalContext, true, tc.frequency, tc.journals) != tc.expect {
evalContext.Rule.State = tc.newState
nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}
if nb.ShouldNotify(evalContext.Ctx, evalContext, tc.state) != tc.expect {
t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect)
}
}
}
func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) {
Convey("base notifier", t, func() {
bus.ClearBusHandlers()
notifier := NewNotifierBase(&m.AlertNotification{
Id: 1,
Name: "name",
Type: "email",
Settings: simplejson.New(),
})
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{})
Convey("should not notify query returns error", func() {
bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error {
return errors.New("some kind of error unknown error")
})
if notifier.ShouldNotify(context.Background(), evalContext) {
t.Errorf("should not send notifications when query returns error")
}
})
})
}
func TestBaseNotifier(t *testing.T) {
Convey("default constructor for notifiers", t, func() {
bJson := simplejson.New()
pkg/services/alerting/result_handler.go
+6 -13
View File
@@ -67,6 +67,12 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
}
handler.log.Error("Failed to save state", "error", err)
} else {
// StateChanges is used for de duping alert notifications
// when two servers are raising. This makes sure that the server
// with the last state change always sends a notification.
evalContext.Rule.StateChanges = cmd.Result.StateChanges
}
// save annotation
@@ -88,19 +94,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
}
}
if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK {
for _, notifierId := range evalContext.Rule.Notifications {
cmd := &m.CleanNotificationJournalCommand{
AlertId: evalContext.Rule.Id,
NotifierId: notifierId,
OrgId: evalContext.Rule.OrgId,
}
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err)
}
}
}
handler.notifier.SendIfNeeded(evalContext)
return nil
}
pkg/services/alerting/rule.go
+3
View File
@@ -23,6 +23,8 @@ type Rule struct {
State m.AlertStateType
Conditions []Condition
Notifications []int64
StateChanges int64
}
type ValidationError struct {
@@ -100,6 +102,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.State = ruleDef.State
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
model.StateChanges = ruleDef.StateChanges
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v)
pkg/services/alerting/test_notification.go
+1 -1
View File
@@ -39,7 +39,7 @@ func handleNotificationTestCommand(cmd *NotificationTestCommand) error {
return err
}
return notifier.sendNotifications(createTestEvalContext(cmd), []Notifier{notifiers})
return notifier.sendNotifications(createTestEvalContext(cmd), notifierStateSlice{{notifier: notifiers}})
}
func createTestEvalContext(cmd *NotificationTestCommand) *EvalContext {