From ff79f80685fb4c17b3fadb239e22c018dddc8902 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 26 Sep 2018 17:26:02 +0200 Subject: [PATCH 01/28] initial rename refactoring --- pkg/models/alert_notifications.go | 17 ++++---- pkg/services/alerting/notifier.go | 39 +++++++++---------- pkg/services/alerting/notifiers/base.go | 19 ++++----- pkg/services/alerting/notifiers/base_test.go | 27 ++++++------- pkg/services/alerting/result_handler.go | 13 ------- pkg/services/sqlstore/alert_notification.go | 18 ++------- .../sqlstore/alert_notification_test.go | 33 +++------------- pkg/services/sqlstore/migrations/alert_mig.go | 21 ++++++++++ 8 files changed, 81 insertions(+), 106 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index b90b3d36ced..419f5048af4 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -76,33 +76,36 @@ type GetAllAlertNotificationsQuery struct { Result []*AlertNotification } -type AlertNotificationJournal struct { +type AlertNotificationState struct { Id int64 OrgId int64 AlertId int64 NotifierId int64 SentAt int64 - Success bool + State string + Version int64 } -type RecordNotificationJournalCommand struct { +type UpdateAlertNotificationStateCommand struct { OrgId int64 AlertId int64 NotifierId int64 SentAt int64 - Success bool + State bool } -type GetLatestNotificationQuery struct { +type GetNotificationStateQuery struct { OrgId int64 AlertId int64 NotifierId int64 - Result []AlertNotificationJournal + Result *AlertNotificationState } -type CleanNotificationJournalCommand struct { +type InsertAlertNotificationCommand struct { OrgId int64 AlertId int64 NotifierId int64 + SentAt int64 + State string } diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index cbad5cbfdcf..ec448a8c1b0 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -68,30 +66,31 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi // Verify that we can send the notification again // but this time within the same transaction. - if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) { - return nil - } + // if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) { + // return nil + // } - n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + // n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) + // metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - //send notification - success := not.Notify(evalContext) == nil + // //send notification + // // success := not.Notify(evalContext) == nil - if evalContext.IsTestRun { - return nil - } + // if evalContext.IsTestRun { + // return nil + // } //write result to db. - cmd := &m.RecordNotificationJournalCommand{ - OrgId: evalContext.Rule.OrgId, - AlertId: evalContext.Rule.Id, - NotifierId: not.GetNotifierId(), - SentAt: time.Now().Unix(), - Success: success, - } + // cmd := &m.RecordNotificationJournalCommand{ + // OrgId: evalContext.Rule.OrgId, + // AlertId: evalContext.Rule.Id, + // NotifierId: not.GetNotifierId(), + // SentAt: time.Now().Unix(), + // Success: success, + // } - return bus.DispatchCtx(ctx, cmd) + // return bus.DispatchCtx(ctx, cmd) + return nil }) if err != nil { diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 24daa02bce8..f71a41235f7 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -42,22 +42,23 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase { } } -func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, journals []models.AlertNotificationJournal) bool { +func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, notificationState *models.AlertNotificationState) bool { // Only notify on state change. if context.PrevAlertState == context.Rule.State && !sendReminder { return false } // get last successfully sent notification - lastNotify := time.Time{} - for _, j := range journals { - if j.Success { - lastNotify = time.Unix(j.SentAt, 0) - break - } - } + // lastNotify := time.Time{} + // for _, j := range journals { + // if j.Success { + // lastNotify = time.Unix(j.SentAt, 0) + // break + // } + // } // Do not notify if interval has not elapsed + lastNotify := time.Unix(notificationState.SentAt, 0) if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { return false } @@ -77,7 +78,7 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ // ShouldNotify checks this evaluation should send an alert notification func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext) bool { - cmd := &models.GetLatestNotificationQuery{ + cmd := &models.GetNotificationStateQuery{ OrgId: c.Rule.OrgId, AlertId: c.Rule.Id, NotifierId: n.Id, diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 9ea4b82fd54..c14006637a4 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -23,7 +23,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState m.AlertStateType sendReminder bool frequency time.Duration - journals []m.AlertNotificationJournal + journals *m.AlertNotificationState expect bool }{ @@ -32,7 +32,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStatePending, prevState: m.AlertStateOK, sendReminder: false, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: false, }, @@ -41,7 +41,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateAlerting, sendReminder: false, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: true, }, @@ -50,7 +50,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStatePending, sendReminder: false, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: false, }, @@ -59,7 +59,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: false, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: false, }, @@ -68,7 +68,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateAlerting, sendReminder: true, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: true, }, @@ -77,7 +77,7 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: true, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: false, }, @@ -87,7 +87,7 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - journals: []m.AlertNotificationJournal{}, + journals: &m.AlertNotificationState{}, expect: true, }, @@ -97,9 +97,7 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - journals: []m.AlertNotificationJournal{ - {SentAt: tnow.Add(-time.Minute).Unix(), Success: true}, - }, + journals: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, expect: false, }, @@ -110,10 +108,7 @@ func TestShouldSendAlertNotification(t *testing.T) { frequency: time.Minute * 10, sendReminder: true, expect: true, - journals: []m.AlertNotificationJournal{ - {SentAt: tnow.Add(-time.Minute).Unix(), Success: false}, // recent failed notification - {SentAt: tnow.Add(-time.Hour).Unix(), Success: true}, // old successful notification - }, + journals: &m.AlertNotificationState{SentAt: tnow.Add(-time.Hour).Unix()}, }, } @@ -142,7 +137,7 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{}) Convey("should not notify query returns error", func() { - bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error { + bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetNotificationStateQuery) error { return errors.New("some kind of error unknown error") }) diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 893cca948f9..e2c70de0e28 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,19 +88,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } - if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { - for _, notifierId := range evalContext.Rule.Notifications { - cmd := &m.CleanNotificationJournalCommand{ - AlertId: evalContext.Rule.Id, - NotifierId: notifierId, - OrgId: evalContext.Rule.OrgId, - } - if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { - handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) - } - } - } - handler.notifier.SendIfNeeded(evalContext) return nil } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index df247e6891d..ece06614d4a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -20,7 +20,6 @@ func init() { bus.AddHandler("sql", GetAllAlertNotifications) bus.AddHandlerCtx("sql", RecordNotificationJournal) bus.AddHandlerCtx("sql", GetLatestNotification) - bus.AddHandlerCtx("sql", CleanNotificationJournal) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -229,14 +228,13 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { }) } -func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJournalCommand) error { +func RecordNotificationJournal(ctx context.Context, cmd *m.UpdateAlertNotificationStateCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - journalEntry := &m.AlertNotificationJournal{ + journalEntry := &m.AlertNotificationState{ OrgId: cmd.OrgId, AlertId: cmd.AlertId, NotifierId: cmd.NotifierId, SentAt: cmd.SentAt, - Success: cmd.Success, } _, err := sess.Insert(journalEntry) @@ -244,9 +242,9 @@ func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJou }) } -func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { +func GetLatestNotification(ctx context.Context, cmd *m.GetNotificationStateQuery) error { return withDbSession(ctx, func(sess *DBSession) error { - nj := []m.AlertNotificationJournal{} + nj := &m.AlertNotificationState{} err := sess.Desc("alert_notification_journal.sent_at"). Where("alert_notification_journal.org_id = ?", cmd.OrgId). @@ -262,11 +260,3 @@ func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuer return nil }) } - -func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error { - return inTransactionCtx(ctx, func(sess *DBSession) error { - sql := "DELETE FROM alert_notification_journal WHERE alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?" - _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) - return err - }) -} diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 1e3df45b5cf..742d459e1ca 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -20,17 +20,17 @@ func TestAlertNotificationSQLAccess(t *testing.T) { var notifierId int64 = 10 Convey("Getting last journal should raise error if no one exists", func() { - query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} - GetLatestNotification(context.Background(), query) - So(len(query.Result), ShouldEqual, 0) + query := &m.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} + err := GetLatestNotification(context.Background(), query) + So(err, ShouldNotBeNil) // recording an journal entry in another org to make sure org filter works as expected. - journalInOtherOrg := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: 10, Success: true, SentAt: 1} - err := RecordNotificationJournal(context.Background(), journalInOtherOrg) + journalInOtherOrg := &m.UpdateAlertNotificationStateCommand{AlertId: alertId, NotifierId: notifierId, OrgId: 10, SentAt: 1} + err = RecordNotificationJournal(context.Background(), journalInOtherOrg) So(err, ShouldBeNil) Convey("should be able to record two journaling events", func() { - createCmd := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, Success: true, SentAt: 1} + createCmd := &m.UpdateAlertNotificationStateCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, SentAt: 1} err := RecordNotificationJournal(context.Background(), createCmd) So(err, ShouldBeNil) @@ -39,27 +39,6 @@ func TestAlertNotificationSQLAccess(t *testing.T) { err = RecordNotificationJournal(context.Background(), createCmd) So(err, ShouldBeNil) - - Convey("get last journaling event", func() { - err := GetLatestNotification(context.Background(), query) - So(err, ShouldBeNil) - So(len(query.Result), ShouldEqual, 2) - last := query.Result[0] - So(last.SentAt, ShouldEqual, 1001) - - Convey("be able to clear all journaling for an notifier", func() { - cmd := &m.CleanNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId} - err := CleanNotificationJournal(context.Background(), cmd) - So(err, ShouldBeNil) - - Convey("querying for last journaling should return no journal entries", func() { - query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} - err := GetLatestNotification(context.Background(), query) - So(err, ShouldBeNil) - So(len(query.Result), ShouldEqual, 0) - }) - }) - }) }) }) }) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index e27e64c6124..d6886f6ecae 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -107,4 +107,25 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) + + mg.AddMigration("drop alert_notification_journal", NewDropTableMigration("alert_notification_journal")) + + alert_notification_state := Table{ + Name: "alert_notification_state", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "sent_at", Type: DB_BigInt, Nullable: false}, + {Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false}, + {Name: "version", Type: DB_BigInt, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + }, + } + + mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) + mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", NewAddIndexMigration(alert_notification_state, notification_journal.Indices[0])) } From 3fab616239aef644e416a75a8db0f67beb3d32be Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 27 Sep 2018 11:14:44 +0200 Subject: [PATCH 02/28] implement sql queries for transactional alert reminders --- pkg/models/alert_notifications.go | 26 +++--- pkg/services/sqlstore/alert_notification.go | 61 +++++++++++--- .../sqlstore/alert_notification_test.go | 80 +++++++++++-------- pkg/services/sqlstore/migrations/alert_mig.go | 5 +- 4 files changed, 118 insertions(+), 54 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 419f5048af4..54220e7d120 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -8,8 +8,17 @@ import ( ) var ( - ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") - ErrJournalingNotFound = errors.New("alert notification journaling not found") + ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") + ErrAlertNotificationStateNotFound = errors.New("alert notification state not found") + ErrAlertNotificationStateVersionConflict = errors.New("alert notification state update version conflict") + ErrAlertNotificationStateAllreadyExist = errors.New("alert notification state allready exists.") +) + +type AlertNotificationStateType string + +var ( + AlertNotificationStatePending = AlertNotificationStateType("pending") + AlertNotificationStateCompleted = AlertNotificationStateType("completed") ) type AlertNotification struct { @@ -82,16 +91,15 @@ type AlertNotificationState struct { AlertId int64 NotifierId int64 SentAt int64 - State string + State AlertNotificationStateType Version int64 } type UpdateAlertNotificationStateCommand struct { - OrgId int64 - AlertId int64 - NotifierId int64 - SentAt int64 - State bool + Id int64 + SentAt int64 + State AlertNotificationStateType + Version int64 } type GetNotificationStateQuery struct { @@ -107,5 +115,5 @@ type InsertAlertNotificationCommand struct { AlertId int64 NotifierId int64 SentAt int64 - State string + State AlertNotificationStateType } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index ece06614d4a..4a115bcd788 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -18,8 +18,8 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) - bus.AddHandlerCtx("sql", RecordNotificationJournal) - bus.AddHandlerCtx("sql", GetLatestNotification) + bus.AddHandlerCtx("sql", InsertAlertNotificationState) + bus.AddHandlerCtx("sql", GetAlertNotificationState) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -228,34 +228,73 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { }) } -func RecordNotificationJournal(ctx context.Context, cmd *m.UpdateAlertNotificationStateCommand) error { +func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotificationCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - journalEntry := &m.AlertNotificationState{ + notificationState := &m.AlertNotificationState{ OrgId: cmd.OrgId, AlertId: cmd.AlertId, NotifierId: cmd.NotifierId, SentAt: cmd.SentAt, + State: cmd.State, + } + + _, err := sess.Insert(notificationState) + + if err == nil { + return nil + } + + if strings.HasPrefix(err.Error(), "UNIQUE constraint failed") { + return m.ErrAlertNotificationStateAllreadyExist } - _, err := sess.Insert(journalEntry) return err }) } -func GetLatestNotification(ctx context.Context, cmd *m.GetNotificationStateQuery) error { +func UpdateAlertNotificationState(ctx context.Context, cmd *m.UpdateAlertNotificationStateCommand) error { + return withDbSession(ctx, func(sess *DBSession) error { + sql := `UPDATE alert_notification_state SET + state= ?, + version = ? + WHERE + id = ? AND + version = ? + ` + + res, err := sess.Exec(sql, cmd.State, cmd.Version+1, cmd.Id, cmd.Version) + if err != nil { + return err + } + + affected, _ := res.RowsAffected() + + if affected == 0 { + return m.ErrAlertNotificationStateVersionConflict + } + + return nil + }) +} + +func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQuery) error { return withDbSession(ctx, func(sess *DBSession) error { nj := &m.AlertNotificationState{} - err := sess.Desc("alert_notification_journal.sent_at"). - Where("alert_notification_journal.org_id = ?", cmd.OrgId). - Where("alert_notification_journal.alert_id = ?", cmd.AlertId). - Where("alert_notification_journal.notifier_id = ?", cmd.NotifierId). - Find(&nj) + exist, err := sess.Desc("alert_notification_state.sent_at"). + Where("alert_notification_state.org_id = ?", cmd.OrgId). + Where("alert_notification_state.alert_id = ?", cmd.AlertId). + Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). + Get(nj) if err != nil { return err } + if !exist { + return m.ErrAlertNotificationStateNotFound + } + cmd.Result = nj return nil }) diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 742d459e1ca..4b71b0d09dd 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) @@ -14,37 +14,53 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) - Convey("Alert notification journal", func() { + Convey("Alert notification state", func() { var alertId int64 = 7 var orgId int64 = 5 var notifierId int64 = 10 - Convey("Getting last journal should raise error if no one exists", func() { - query := &m.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} - err := GetLatestNotification(context.Background(), query) - So(err, ShouldNotBeNil) + Convey("Getting no existant state returns error", func() { + query := &models.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} + err := GetAlertNotificationState(context.Background(), query) + So(err, ShouldEqual, models.ErrAlertNotificationStateNotFound) + }) - // recording an journal entry in another org to make sure org filter works as expected. - journalInOtherOrg := &m.UpdateAlertNotificationStateCommand{AlertId: alertId, NotifierId: notifierId, OrgId: 10, SentAt: 1} - err = RecordNotificationJournal(context.Background(), journalInOtherOrg) + Convey("Can insert new state for alert notifier", func() { + createCmd := &models.InsertAlertNotificationCommand{ + AlertId: alertId, + NotifierId: notifierId, + OrgId: orgId, + SentAt: 1, + State: models.AlertNotificationStateCompleted, + } + + err := InsertAlertNotificationState(context.Background(), createCmd) So(err, ShouldBeNil) - Convey("should be able to record two journaling events", func() { - createCmd := &m.UpdateAlertNotificationStateCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, SentAt: 1} + err = InsertAlertNotificationState(context.Background(), createCmd) + So(err, ShouldEqual, models.ErrAlertNotificationStateAllreadyExist) - err := RecordNotificationJournal(context.Background(), createCmd) + Convey("should be able to update alert notifier state", func() { + updateCmd := &models.UpdateAlertNotificationStateCommand{ + Id: 1, + SentAt: 1, + State: models.AlertNotificationStatePending, + Version: 0, + } + + err := UpdateAlertNotificationState(context.Background(), updateCmd) So(err, ShouldBeNil) - createCmd.SentAt += 1000 //increase epoch - - err = RecordNotificationJournal(context.Background(), createCmd) - So(err, ShouldBeNil) + Convey("should not be able to update older versions", func() { + err = UpdateAlertNotificationState(context.Background(), updateCmd) + So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + }) }) }) }) Convey("Alert notifications should be empty", func() { - cmd := &m.GetAlertNotificationsQuery{ + cmd := &models.GetAlertNotificationsQuery{ OrgId: 2, Name: "email", } @@ -55,7 +71,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Cannot save alert notifier with send reminder = true", func() { - cmd := &m.CreateAlertNotificationCommand{ + cmd := &models.CreateAlertNotificationCommand{ Name: "ops", Type: "email", OrgId: 1, @@ -65,7 +81,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("and missing frequency", func() { err := CreateAlertNotificationCommand(cmd) - So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound) + So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound) }) Convey("invalid frequency", func() { @@ -77,7 +93,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Cannot update alert notifier with send reminder = false", func() { - cmd := &m.CreateAlertNotificationCommand{ + cmd := &models.CreateAlertNotificationCommand{ Name: "ops update", Type: "email", OrgId: 1, @@ -88,14 +104,14 @@ func TestAlertNotificationSQLAccess(t *testing.T) { err := CreateAlertNotificationCommand(cmd) So(err, ShouldBeNil) - updateCmd := &m.UpdateAlertNotificationCommand{ + updateCmd := &models.UpdateAlertNotificationCommand{ Id: cmd.Result.Id, SendReminder: true, } Convey("and missing frequency", func() { err := UpdateAlertNotification(updateCmd) - So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound) + So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound) }) Convey("invalid frequency", func() { @@ -108,7 +124,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can save Alert Notification", func() { - cmd := &m.CreateAlertNotificationCommand{ + cmd := &models.CreateAlertNotificationCommand{ Name: "ops", Type: "email", OrgId: 1, @@ -130,7 +146,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can update alert notification", func() { - newCmd := &m.UpdateAlertNotificationCommand{ + newCmd := &models.UpdateAlertNotificationCommand{ Name: "NewName", Type: "webhook", OrgId: cmd.Result.OrgId, @@ -146,7 +162,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can update alert notification to disable sending of reminders", func() { - newCmd := &m.UpdateAlertNotificationCommand{ + newCmd := &models.UpdateAlertNotificationCommand{ Name: "NewName", Type: "webhook", OrgId: cmd.Result.OrgId, @@ -161,12 +177,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can search using an array of ids", func() { - cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd1 := models.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd2 := models.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd3 := models.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd4 := models.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + otherOrg := models.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil) So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil) @@ -175,7 +191,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(CreateAlertNotificationCommand(&otherOrg), ShouldBeNil) Convey("search", func() { - query := &m.GetAlertNotificationsToSendQuery{ + query := &models.GetAlertNotificationsToSendQuery{ Ids: []int64{cmd1.Result.Id, cmd2.Result.Id, 112341231}, OrgId: 1, } @@ -186,7 +202,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("all", func() { - query := &m.GetAllAlertNotificationsQuery{ + query := &models.GetAllAlertNotificationsQuery{ OrgId: 1, } diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index d6886f6ecae..877dafcf1e1 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -122,10 +122,11 @@ func addAlertMigrations(mg *Migrator) { {Name: "version", Type: DB_BigInt, Nullable: false}, }, Indices: []*Index{ - {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex}, }, } mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) - mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", NewAddIndexMigration(alert_notification_state, notification_journal.Indices[0])) + mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", + NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0])) } From c5278af6c498020d206eebe601f7dcd23160668b Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 27 Sep 2018 11:33:13 +0200 Subject: [PATCH 03/28] add support for mysql and postgres unique index error codes --- pkg/models/alert_notifications.go | 10 +++-- pkg/services/alerting/notifier.go | 2 + pkg/services/sqlstore/alert_notification.go | 43 ++++++++++++++++--- .../sqlstore/alert_notification_test.go | 17 +++++--- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 54220e7d120..c3c41dc5dd9 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -11,7 +11,7 @@ var ( ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") ErrAlertNotificationStateNotFound = errors.New("alert notification state not found") ErrAlertNotificationStateVersionConflict = errors.New("alert notification state update version conflict") - ErrAlertNotificationStateAllreadyExist = errors.New("alert notification state allready exists.") + ErrAlertNotificationStateAlreadyExist = errors.New("alert notification state already exists.") ) type AlertNotificationStateType string @@ -95,13 +95,17 @@ type AlertNotificationState struct { Version int64 } -type UpdateAlertNotificationStateCommand struct { +type SetAlertNotificationStateToPendingCommand struct { Id int64 SentAt int64 - State AlertNotificationStateType Version int64 } +type SetAlertNotificationStateToCompleteCommand struct { + Id int64 + SentAt int64 +} + type GetNotificationStateQuery struct { OrgId int64 AlertId int64 diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index ec448a8c1b0..941d3df9dc3 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -64,6 +64,8 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi err := bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error { n.log.Debug("trying to send notification", "id", not.GetNotifierId()) + // insert if needed + // Verify that we can send the notification again // but this time within the same transaction. // if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 4a115bcd788..92860c361aa 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -20,6 +20,8 @@ func init() { bus.AddHandler("sql", GetAllAlertNotifications) bus.AddHandlerCtx("sql", InsertAlertNotificationState) bus.AddHandlerCtx("sql", GetAlertNotificationState) + bus.AddHandlerCtx("sql", SetAlertNotificationStateToCompleteCommand) + bus.AddHandlerCtx("sql", SetAlertNotificationStateToPendingCommand) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -244,25 +246,54 @@ func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotific return nil } - if strings.HasPrefix(err.Error(), "UNIQUE constraint failed") { - return m.ErrAlertNotificationStateAllreadyExist + uniqenessIndexFailureCodes := []string{ + "UNIQUE constraint failed", + "pq: duplicate key value violates unique constraint", + "Error 1062: Duplicate entry ", + } + + for _, code := range uniqenessIndexFailureCodes { + if strings.HasPrefix(err.Error(), code) { + return m.ErrAlertNotificationStateAlreadyExist + } } return err }) } -func UpdateAlertNotificationState(ctx context.Context, cmd *m.UpdateAlertNotificationStateCommand) error { +func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { + return withDbSession(ctx, func(sess *DBSession) error { + sql := `UPDATE alert_notification_state SET + state= ? + WHERE + id = ?` + + res, err := sess.Exec(sql, m.AlertNotificationStateCompleted, cmd.Id) + if err != nil { + return err + } + + affected, _ := res.RowsAffected() + + if affected == 0 { + return m.ErrAlertNotificationStateVersionConflict + } + + return nil + }) +} + +func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { return withDbSession(ctx, func(sess *DBSession) error { sql := `UPDATE alert_notification_state SET state= ?, version = ? WHERE id = ? AND - version = ? - ` + version = ?` - res, err := sess.Exec(sql, cmd.State, cmd.Version+1, cmd.Id, cmd.Version) + res, err := sess.Exec(sql, m.AlertNotificationStatePending, cmd.Version+1, cmd.Id, cmd.Version) if err != nil { return err } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 4b71b0d09dd..206c96b5c6a 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -38,23 +38,28 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) err = InsertAlertNotificationState(context.Background(), createCmd) - So(err, ShouldEqual, models.ErrAlertNotificationStateAllreadyExist) + So(err, ShouldEqual, models.ErrAlertNotificationStateAlreadyExist) Convey("should be able to update alert notifier state", func() { - updateCmd := &models.UpdateAlertNotificationStateCommand{ + updateCmd := &models.SetAlertNotificationStateToPendingCommand{ Id: 1, SentAt: 1, - State: models.AlertNotificationStatePending, Version: 0, } - err := UpdateAlertNotificationState(context.Background(), updateCmd) + err := SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) So(err, ShouldBeNil) - Convey("should not be able to update older versions", func() { - err = UpdateAlertNotificationState(context.Background(), updateCmd) + Convey("should not be able to set pending on old version", func() { + err = SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) }) + + Convey("should be able to set state to completed", func() { + cmd := &models.SetAlertNotificationStateToCompleteCommand{Id: 1} + err = SetAlertNotificationStateToCompleteCommand(context.Background(), cmd) + So(err, ShouldBeNil) + }) }) }) }) From d093244282589781ca57ecc9cba1053d3e04a358 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 27 Sep 2018 12:07:43 +0200 Subject: [PATCH 04/28] sqlstore: add support for checking if error is constraint validation error --- Gopkg.lock | 8 +- Gopkg.toml | 4 + pkg/services/sqlstore/alert_notification.go | 26 +- pkg/services/sqlstore/migrator/dialect.go | 2 + .../sqlstore/migrator/mysql_dialect.go | 12 + .../sqlstore/migrator/postgres_dialect.go | 11 + .../sqlstore/migrator/sqlite_dialect.go | 11 + .../github.com/VividCortex/mysqlerr/LICENSE | 21 + .../VividCortex/mysqlerr/mysqlerr.go | 1080 +++++++++++++++++ 9 files changed, 1156 insertions(+), 19 deletions(-) create mode 100644 vendor/github.com/VividCortex/mysqlerr/LICENSE create mode 100644 vendor/github.com/VividCortex/mysqlerr/mysqlerr.go diff --git a/Gopkg.lock b/Gopkg.lock index bd247d691dd..041f784f770 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -19,6 +19,12 @@ packages = ["."] revision = "7677a1d7c1137cd3dd5ba7a076d0c898a1ef4520" +[[projects]] + branch = "master" + name = "github.com/VividCortex/mysqlerr" + packages = ["."] + revision = "6c6b55f8796f578c870b7e19bafb16103bc40095" + [[projects]] name = "github.com/aws/aws-sdk-go" packages = [ @@ -673,6 +679,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "81a37e747b875cf870c1b9486fa3147e704dea7db8ba86f7cb942d3ddc01d3e3" + inputs-digest = "6e9458f912a5f0eb3430b968f1b4dbc4e3b7671b282cf4fe1573419a6d9ba0d4" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 6c91ec37221..c5b4b31cb32 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -203,3 +203,7 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" + +[[constraint]] + name = "github.com/VividCortex/mysqlerr" + branch = "master" diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 92860c361aa..a89e777ede3 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -240,31 +240,21 @@ func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotific State: cmd.State, } - _, err := sess.Insert(notificationState) - - if err == nil { - return nil - } - - uniqenessIndexFailureCodes := []string{ - "UNIQUE constraint failed", - "pq: duplicate key value violates unique constraint", - "Error 1062: Duplicate entry ", - } - - for _, code := range uniqenessIndexFailureCodes { - if strings.HasPrefix(err.Error(), code) { + if _, err := sess.Insert(notificationState); err != nil { + if dialect.IsUniqueConstraintViolation(err) { return m.ErrAlertNotificationStateAlreadyExist } + + return err } - return err + return nil }) } func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - sql := `UPDATE alert_notification_state SET + sql := `UPDATE alert_notification_state SET state= ? WHERE id = ?` @@ -286,8 +276,8 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - sql := `UPDATE alert_notification_state SET - state= ?, + sql := `UPDATE alert_notification_state SET + state= ?, version = ? WHERE id = ? AND diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index 427d102b280..506a01c3ed8 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -44,6 +44,8 @@ type Dialect interface { CleanDB() error NoOpSql() string + + IsUniqueConstraintViolation(err error) bool } func NewDialect(engine *xorm.Engine) Dialect { diff --git a/pkg/services/sqlstore/migrator/mysql_dialect.go b/pkg/services/sqlstore/migrator/mysql_dialect.go index 1ed16871c15..7daa4597430 100644 --- a/pkg/services/sqlstore/migrator/mysql_dialect.go +++ b/pkg/services/sqlstore/migrator/mysql_dialect.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" + "github.com/VividCortex/mysqlerr" + "github.com/go-sql-driver/mysql" "github.com/go-xorm/xorm" ) @@ -125,3 +127,13 @@ func (db *Mysql) CleanDB() error { return nil } + +func (db *Mysql) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(*mysql.MySQLError); ok { + if driverErr.Number == mysqlerr.ER_DUP_ENTRY { + return true + } + } + + return false +} diff --git a/pkg/services/sqlstore/migrator/postgres_dialect.go b/pkg/services/sqlstore/migrator/postgres_dialect.go index eae9ad3ca3f..ab8812a1e26 100644 --- a/pkg/services/sqlstore/migrator/postgres_dialect.go +++ b/pkg/services/sqlstore/migrator/postgres_dialect.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/go-xorm/xorm" + "github.com/lib/pq" ) type Postgres struct { @@ -136,3 +137,13 @@ func (db *Postgres) CleanDB() error { return nil } + +func (db *Postgres) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(*pq.Error); ok { + if driverErr.Code == "23505" { + return true + } + } + + return false +} diff --git a/pkg/services/sqlstore/migrator/sqlite_dialect.go b/pkg/services/sqlstore/migrator/sqlite_dialect.go index 01082b95c88..446e3fcef12 100644 --- a/pkg/services/sqlstore/migrator/sqlite_dialect.go +++ b/pkg/services/sqlstore/migrator/sqlite_dialect.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/go-xorm/xorm" + sqlite3 "github.com/mattn/go-sqlite3" ) type Sqlite3 struct { @@ -82,3 +83,13 @@ func (db *Sqlite3) DropIndexSql(tableName string, index *Index) string { func (db *Sqlite3) CleanDB() error { return nil } + +func (db *Sqlite3) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(sqlite3.Error); ok { + if driverErr.ExtendedCode == sqlite3.ErrConstraintUnique { + return true + } + } + + return false +} diff --git a/vendor/github.com/VividCortex/mysqlerr/LICENSE b/vendor/github.com/VividCortex/mysqlerr/LICENSE new file mode 100644 index 00000000000..92bb301a02f --- /dev/null +++ b/vendor/github.com/VividCortex/mysqlerr/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 VividCortex + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/VividCortex/mysqlerr/mysqlerr.go b/vendor/github.com/VividCortex/mysqlerr/mysqlerr.go new file mode 100644 index 00000000000..ce62e7d5041 --- /dev/null +++ b/vendor/github.com/VividCortex/mysqlerr/mysqlerr.go @@ -0,0 +1,1080 @@ +package mysqlerr + +const ( + ER_HASHCHK = 1000 + ER_NISAMCHK = 1001 + ER_NO = 1002 + ER_YES = 1003 + ER_CANT_CREATE_FILE = 1004 + ER_CANT_CREATE_TABLE = 1005 + ER_CANT_CREATE_DB = 1006 + ER_DB_CREATE_EXISTS = 1007 + ER_DB_DROP_EXISTS = 1008 + ER_DB_DROP_DELETE = 1009 + ER_DB_DROP_RMDIR = 1010 + ER_CANT_DELETE_FILE = 1011 + ER_CANT_FIND_SYSTEM_REC = 1012 + ER_CANT_GET_STAT = 1013 + ER_CANT_GET_WD = 1014 + ER_CANT_LOCK = 1015 + ER_CANT_OPEN_FILE = 1016 + ER_FILE_NOT_FOUND = 1017 + ER_CANT_READ_DIR = 1018 + ER_CANT_SET_WD = 1019 + ER_CHECKREAD = 1020 + ER_DISK_FULL = 1021 + ER_DUP_KEY = 1022 + ER_ERROR_ON_CLOSE = 1023 + ER_ERROR_ON_READ = 1024 + ER_ERROR_ON_RENAME = 1025 + ER_ERROR_ON_WRITE = 1026 + ER_FILE_USED = 1027 + ER_FILSORT_ABORT = 1028 + ER_FORM_NOT_FOUND = 1029 + ER_GET_ERRNO = 1030 + ER_ILLEGAL_HA = 1031 + ER_KEY_NOT_FOUND = 1032 + ER_NOT_FORM_FILE = 1033 + ER_NOT_KEYFILE = 1034 + ER_OLD_KEYFILE = 1035 + ER_OPEN_AS_READONLY = 1036 + ER_OUTOFMEMORY = 1037 + ER_OUT_OF_SORTMEMORY = 1038 + ER_UNEXPECTED_EOF = 1039 + ER_CON_COUNT_ERROR = 1040 + ER_OUT_OF_RESOURCES = 1041 + ER_BAD_HOST_ERROR = 1042 + ER_HANDSHAKE_ERROR = 1043 + ER_DBACCESS_DENIED_ERROR = 1044 + ER_ACCESS_DENIED_ERROR = 1045 + ER_NO_DB_ERROR = 1046 + ER_UNKNOWN_COM_ERROR = 1047 + ER_BAD_NULL_ERROR = 1048 + ER_BAD_DB_ERROR = 1049 + ER_TABLE_EXISTS_ERROR = 1050 + ER_BAD_TABLE_ERROR = 1051 + ER_NON_UNIQ_ERROR = 1052 + ER_SERVER_SHUTDOWN = 1053 + ER_BAD_FIELD_ERROR = 1054 + ER_WRONG_FIELD_WITH_GROUP = 1055 + ER_WRONG_GROUP_FIELD = 1056 + ER_WRONG_SUM_SELECT = 1057 + ER_WRONG_VALUE_COUNT = 1058 + ER_TOO_LONG_IDENT = 1059 + ER_DUP_FIELDNAME = 1060 + ER_DUP_KEYNAME = 1061 + ER_DUP_ENTRY = 1062 + ER_WRONG_FIELD_SPEC = 1063 + ER_PARSE_ERROR = 1064 + ER_EMPTY_QUERY = 1065 + ER_NONUNIQ_TABLE = 1066 + ER_INVALID_DEFAULT = 1067 + ER_MULTIPLE_PRI_KEY = 1068 + ER_TOO_MANY_KEYS = 1069 + ER_TOO_MANY_KEY_PARTS = 1070 + ER_TOO_LONG_KEY = 1071 + ER_KEY_COLUMN_DOES_NOT_EXITS = 1072 + ER_BLOB_USED_AS_KEY = 1073 + ER_TOO_BIG_FIELDLENGTH = 1074 + ER_WRONG_AUTO_KEY = 1075 + ER_READY = 1076 + ER_NORMAL_SHUTDOWN = 1077 + ER_GOT_SIGNAL = 1078 + ER_SHUTDOWN_COMPLETE = 1079 + ER_FORCING_CLOSE = 1080 + ER_IPSOCK_ERROR = 1081 + ER_NO_SUCH_INDEX = 1082 + ER_WRONG_FIELD_TERMINATORS = 1083 + ER_BLOBS_AND_NO_TERMINATED = 1084 + ER_TEXTFILE_NOT_READABLE = 1085 + ER_FILE_EXISTS_ERROR = 1086 + ER_LOAD_INFO = 1087 + ER_ALTER_INFO = 1088 + ER_WRONG_SUB_KEY = 1089 + ER_CANT_REMOVE_ALL_FIELDS = 1090 + ER_CANT_DROP_FIELD_OR_KEY = 1091 + ER_INSERT_INFO = 1092 + ER_UPDATE_TABLE_USED = 1093 + ER_NO_SUCH_THREAD = 1094 + ER_KILL_DENIED_ERROR = 1095 + ER_NO_TABLES_USED = 1096 + ER_TOO_BIG_SET = 1097 + ER_NO_UNIQUE_LOGFILE = 1098 + ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099 + ER_TABLE_NOT_LOCKED = 1100 + ER_BLOB_CANT_HAVE_DEFAULT = 1101 + ER_WRONG_DB_NAME = 1102 + ER_WRONG_TABLE_NAME = 1103 + ER_TOO_BIG_SELECT = 1104 + ER_UNKNOWN_ERROR = 1105 + ER_UNKNOWN_PROCEDURE = 1106 + ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107 + ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108 + ER_UNKNOWN_TABLE = 1109 + ER_FIELD_SPECIFIED_TWICE = 1110 + ER_INVALID_GROUP_FUNC_USE = 1111 + ER_UNSUPPORTED_EXTENSION = 1112 + ER_TABLE_MUST_HAVE_COLUMNS = 1113 + ER_RECORD_FILE_FULL = 1114 + ER_UNKNOWN_CHARACTER_SET = 1115 + ER_TOO_MANY_TABLES = 1116 + ER_TOO_MANY_FIELDS = 1117 + ER_TOO_BIG_ROWSIZE = 1118 + ER_STACK_OVERRUN = 1119 + ER_WRONG_OUTER_JOIN = 1120 + ER_NULL_COLUMN_IN_INDEX = 1121 + ER_CANT_FIND_UDF = 1122 + ER_CANT_INITIALIZE_UDF = 1123 + ER_UDF_NO_PATHS = 1124 + ER_UDF_EXISTS = 1125 + ER_CANT_OPEN_LIBRARY = 1126 + ER_CANT_FIND_DL_ENTRY = 1127 + ER_FUNCTION_NOT_DEFINED = 1128 + ER_HOST_IS_BLOCKED = 1129 + ER_HOST_NOT_PRIVILEGED = 1130 + ER_PASSWORD_ANONYMOUS_USER = 1131 + ER_PASSWORD_NOT_ALLOWED = 1132 + ER_PASSWORD_NO_MATCH = 1133 + ER_UPDATE_INFO = 1134 + ER_CANT_CREATE_THREAD = 1135 + ER_WRONG_VALUE_COUNT_ON_ROW = 1136 + ER_CANT_REOPEN_TABLE = 1137 + ER_INVALID_USE_OF_NULL = 1138 + ER_REGEXP_ERROR = 1139 + ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140 + ER_NONEXISTING_GRANT = 1141 + ER_TABLEACCESS_DENIED_ERROR = 1142 + ER_COLUMNACCESS_DENIED_ERROR = 1143 + ER_ILLEGAL_GRANT_FOR_TABLE = 1144 + ER_GRANT_WRONG_HOST_OR_USER = 1145 + ER_NO_SUCH_TABLE = 1146 + ER_NONEXISTING_TABLE_GRANT = 1147 + ER_NOT_ALLOWED_COMMAND = 1148 + ER_SYNTAX_ERROR = 1149 + ER_DELAYED_CANT_CHANGE_LOCK = 1150 + ER_TOO_MANY_DELAYED_THREADS = 1151 + ER_ABORTING_CONNECTION = 1152 + ER_NET_PACKET_TOO_LARGE = 1153 + ER_NET_READ_ERROR_FROM_PIPE = 1154 + ER_NET_FCNTL_ERROR = 1155 + ER_NET_PACKETS_OUT_OF_ORDER = 1156 + ER_NET_UNCOMPRESS_ERROR = 1157 + ER_NET_READ_ERROR = 1158 + ER_NET_READ_INTERRUPTED = 1159 + ER_NET_ERROR_ON_WRITE = 1160 + ER_NET_WRITE_INTERRUPTED = 1161 + ER_TOO_LONG_STRING = 1162 + ER_TABLE_CANT_HANDLE_BLOB = 1163 + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164 + ER_DELAYED_INSERT_TABLE_LOCKED = 1165 + ER_WRONG_COLUMN_NAME = 1166 + ER_WRONG_KEY_COLUMN = 1167 + ER_WRONG_MRG_TABLE = 1168 + ER_DUP_UNIQUE = 1169 + ER_BLOB_KEY_WITHOUT_LENGTH = 1170 + ER_PRIMARY_CANT_HAVE_NULL = 1171 + ER_TOO_MANY_ROWS = 1172 + ER_REQUIRES_PRIMARY_KEY = 1173 + ER_NO_RAID_COMPILED = 1174 + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175 + ER_KEY_DOES_NOT_EXITS = 1176 + ER_CHECK_NO_SUCH_TABLE = 1177 + ER_CHECK_NOT_IMPLEMENTED = 1178 + ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179 + ER_ERROR_DURING_COMMIT = 1180 + ER_ERROR_DURING_ROLLBACK = 1181 + ER_ERROR_DURING_FLUSH_LOGS = 1182 + ER_ERROR_DURING_CHECKPOINT = 1183 + ER_NEW_ABORTING_CONNECTION = 1184 + ER_DUMP_NOT_IMPLEMENTED = 1185 + ER_FLUSH_MASTER_BINLOG_CLOSED = 1186 + ER_INDEX_REBUILD = 1187 + ER_MASTER = 1188 + ER_MASTER_NET_READ = 1189 + ER_MASTER_NET_WRITE = 1190 + ER_FT_MATCHING_KEY_NOT_FOUND = 1191 + ER_LOCK_OR_ACTIVE_TRANSACTION = 1192 + ER_UNKNOWN_SYSTEM_VARIABLE = 1193 + ER_CRASHED_ON_USAGE = 1194 + ER_CRASHED_ON_REPAIR = 1195 + ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196 + ER_TRANS_CACHE_FULL = 1197 + ER_SLAVE_MUST_STOP = 1198 + ER_SLAVE_NOT_RUNNING = 1199 + ER_BAD_SLAVE = 1200 + ER_MASTER_INFO = 1201 + ER_SLAVE_THREAD = 1202 + ER_TOO_MANY_USER_CONNECTIONS = 1203 + ER_SET_CONSTANTS_ONLY = 1204 + ER_LOCK_WAIT_TIMEOUT = 1205 + ER_LOCK_TABLE_FULL = 1206 + ER_READ_ONLY_TRANSACTION = 1207 + ER_DROP_DB_WITH_READ_LOCK = 1208 + ER_CREATE_DB_WITH_READ_LOCK = 1209 + ER_WRONG_ARGUMENTS = 1210 + ER_NO_PERMISSION_TO_CREATE_USER = 1211 + ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212 + ER_LOCK_DEADLOCK = 1213 + ER_TABLE_CANT_HANDLE_FT = 1214 + ER_CANNOT_ADD_FOREIGN = 1215 + ER_NO_REFERENCED_ROW = 1216 + ER_ROW_IS_REFERENCED = 1217 + ER_CONNECT_TO_MASTER = 1218 + ER_QUERY_ON_MASTER = 1219 + ER_ERROR_WHEN_EXECUTING_COMMAND = 1220 + ER_WRONG_USAGE = 1221 + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222 + ER_CANT_UPDATE_WITH_READLOCK = 1223 + ER_MIXING_NOT_ALLOWED = 1224 + ER_DUP_ARGUMENT = 1225 + ER_USER_LIMIT_REACHED = 1226 + ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227 + ER_LOCAL_VARIABLE = 1228 + ER_GLOBAL_VARIABLE = 1229 + ER_NO_DEFAULT = 1230 + ER_WRONG_VALUE_FOR_VAR = 1231 + ER_WRONG_TYPE_FOR_VAR = 1232 + ER_VAR_CANT_BE_READ = 1233 + ER_CANT_USE_OPTION_HERE = 1234 + ER_NOT_SUPPORTED_YET = 1235 + ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236 + ER_SLAVE_IGNORED_TABLE = 1237 + ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238 + ER_WRONG_FK_DEF = 1239 + ER_KEY_REF_DO_NOT_MATCH_TABLE_REF = 1240 + ER_OPERAND_COLUMNS = 1241 + ER_SUBQUERY_NO_1_ROW = 1242 + ER_UNKNOWN_STMT_HANDLER = 1243 + ER_CORRUPT_HELP_DB = 1244 + ER_CYCLIC_REFERENCE = 1245 + ER_AUTO_CONVERT = 1246 + ER_ILLEGAL_REFERENCE = 1247 + ER_DERIVED_MUST_HAVE_ALIAS = 1248 + ER_SELECT_REDUCED = 1249 + ER_TABLENAME_NOT_ALLOWED_HERE = 1250 + ER_NOT_SUPPORTED_AUTH_MODE = 1251 + ER_SPATIAL_CANT_HAVE_NULL = 1252 + ER_COLLATION_CHARSET_MISMATCH = 1253 + ER_SLAVE_WAS_RUNNING = 1254 + ER_SLAVE_WAS_NOT_RUNNING = 1255 + ER_TOO_BIG_FOR_UNCOMPRESS = 1256 + ER_ZLIB_Z_MEM_ERROR = 1257 + ER_ZLIB_Z_BUF_ERROR = 1258 + ER_ZLIB_Z_DATA_ERROR = 1259 + ER_CUT_VALUE_GROUP_CONCAT = 1260 + ER_WARN_TOO_FEW_RECORDS = 1261 + ER_WARN_TOO_MANY_RECORDS = 1262 + ER_WARN_NULL_TO_NOTNULL = 1263 + ER_WARN_DATA_OUT_OF_RANGE = 1264 + WARN_DATA_TRUNCATED = 1265 + ER_WARN_USING_OTHER_HANDLER = 1266 + ER_CANT_AGGREGATE_2COLLATIONS = 1267 + ER_DROP_USER = 1268 + ER_REVOKE_GRANTS = 1269 + ER_CANT_AGGREGATE_3COLLATIONS = 1270 + ER_CANT_AGGREGATE_NCOLLATIONS = 1271 + ER_VARIABLE_IS_NOT_STRUCT = 1272 + ER_UNKNOWN_COLLATION = 1273 + ER_SLAVE_IGNORED_SSL_PARAMS = 1274 + ER_SERVER_IS_IN_SECURE_AUTH_MODE = 1275 + ER_WARN_FIELD_RESOLVED = 1276 + ER_BAD_SLAVE_UNTIL_COND = 1277 + ER_MISSING_SKIP_SLAVE = 1278 + ER_UNTIL_COND_IGNORED = 1279 + ER_WRONG_NAME_FOR_INDEX = 1280 + ER_WRONG_NAME_FOR_CATALOG = 1281 + ER_WARN_QC_RESIZE = 1282 + ER_BAD_FT_COLUMN = 1283 + ER_UNKNOWN_KEY_CACHE = 1284 + ER_WARN_HOSTNAME_WONT_WORK = 1285 + ER_UNKNOWN_STORAGE_ENGINE = 1286 + ER_WARN_DEPRECATED_SYNTAX = 1287 + ER_NON_UPDATABLE_TABLE = 1288 + ER_FEATURE_DISABLED = 1289 + ER_OPTION_PREVENTS_STATEMENT = 1290 + ER_DUPLICATED_VALUE_IN_TYPE = 1291 + ER_TRUNCATED_WRONG_VALUE = 1292 + ER_TOO_MUCH_AUTO_TIMESTAMP_COLS = 1293 + ER_INVALID_ON_UPDATE = 1294 + ER_UNSUPPORTED_PS = 1295 + ER_GET_ERRMSG = 1296 + ER_GET_TEMPORARY_ERRMSG = 1297 + ER_UNKNOWN_TIME_ZONE = 1298 + ER_WARN_INVALID_TIMESTAMP = 1299 + ER_INVALID_CHARACTER_STRING = 1300 + ER_WARN_ALLOWED_PACKET_OVERFLOWED = 1301 + ER_CONFLICTING_DECLARATIONS = 1302 + ER_SP_NO_RECURSIVE_CREATE = 1303 + ER_SP_ALREADY_EXISTS = 1304 + ER_SP_DOES_NOT_EXIST = 1305 + ER_SP_DROP_FAILED = 1306 + ER_SP_STORE_FAILED = 1307 + ER_SP_LILABEL_MISMATCH = 1308 + ER_SP_LABEL_REDEFINE = 1309 + ER_SP_LABEL_MISMATCH = 1310 + ER_SP_UNINIT_VAR = 1311 + ER_SP_BADSELECT = 1312 + ER_SP_BADRETURN = 1313 + ER_SP_BADSTATEMENT = 1314 + ER_UPDATE_LOG_DEPRECATED_IGNORED = 1315 + ER_UPDATE_LOG_DEPRECATED_TRANSLATED = 1316 + ER_QUERY_INTERRUPTED = 1317 + ER_SP_WRONG_NO_OF_ARGS = 1318 + ER_SP_COND_MISMATCH = 1319 + ER_SP_NORETURN = 1320 + ER_SP_NORETURNEND = 1321 + ER_SP_BAD_CURSOR_QUERY = 1322 + ER_SP_BAD_CURSOR_SELECT = 1323 + ER_SP_CURSOR_MISMATCH = 1324 + ER_SP_CURSOR_ALREADY_OPEN = 1325 + ER_SP_CURSOR_NOT_OPEN = 1326 + ER_SP_UNDECLARED_VAR = 1327 + ER_SP_WRONG_NO_OF_FETCH_ARGS = 1328 + ER_SP_FETCH_NO_DATA = 1329 + ER_SP_DUP_PARAM = 1330 + ER_SP_DUP_VAR = 1331 + ER_SP_DUP_COND = 1332 + ER_SP_DUP_CURS = 1333 + ER_SP_CANT_ALTER = 1334 + ER_SP_SUBSELECT_NYI = 1335 + ER_STMT_NOT_ALLOWED_IN_SF_OR_TRG = 1336 + ER_SP_VARCOND_AFTER_CURSHNDLR = 1337 + ER_SP_CURSOR_AFTER_HANDLER = 1338 + ER_SP_CASE_NOT_FOUND = 1339 + ER_FPARSER_TOO_BIG_FILE = 1340 + ER_FPARSER_BAD_HEADER = 1341 + ER_FPARSER_EOF_IN_COMMENT = 1342 + ER_FPARSER_ERROR_IN_PARAMETER = 1343 + ER_FPARSER_EOF_IN_UNKNOWN_PARAMETER = 1344 + ER_VIEW_NO_EXPLAIN = 1345 + ER_FRM_UNKNOWN_TYPE = 1346 + ER_WRONG_OBJECT = 1347 + ER_NONUPDATEABLE_COLUMN = 1348 + ER_VIEW_SELECT_DERIVED = 1349 + ER_VIEW_SELECT_CLAUSE = 1350 + ER_VIEW_SELECT_VARIABLE = 1351 + ER_VIEW_SELECT_TMPTABLE = 1352 + ER_VIEW_WRONG_LIST = 1353 + ER_WARN_VIEW_MERGE = 1354 + ER_WARN_VIEW_WITHOUT_KEY = 1355 + ER_VIEW_INVALID = 1356 + ER_SP_NO_DROP_SP = 1357 + ER_SP_GOTO_IN_HNDLR = 1358 + ER_TRG_ALREADY_EXISTS = 1359 + ER_TRG_DOES_NOT_EXIST = 1360 + ER_TRG_ON_VIEW_OR_TEMP_TABLE = 1361 + ER_TRG_CANT_CHANGE_ROW = 1362 + ER_TRG_NO_SUCH_ROW_IN_TRG = 1363 + ER_NO_DEFAULT_FOR_FIELD = 1364 + ER_DIVISION_BY_ZERO = 1365 + ER_TRUNCATED_WRONG_VALUE_FOR_FIELD = 1366 + ER_ILLEGAL_VALUE_FOR_TYPE = 1367 + ER_VIEW_NONUPD_CHECK = 1368 + ER_VIEW_CHECK_FAILED = 1369 + ER_PROCACCESS_DENIED_ERROR = 1370 + ER_RELAY_LOG_FAIL = 1371 + ER_PASSWD_LENGTH = 1372 + ER_UNKNOWN_TARGET_BINLOG = 1373 + ER_IO_ERR_LOG_INDEX_READ = 1374 + ER_BINLOG_PURGE_PROHIBITED = 1375 + ER_FSEEK_FAIL = 1376 + ER_BINLOG_PURGE_FATAL_ERR = 1377 + ER_LOG_IN_USE = 1378 + ER_LOG_PURGE_UNKNOWN_ERR = 1379 + ER_RELAY_LOG_INIT = 1380 + ER_NO_BINARY_LOGGING = 1381 + ER_RESERVED_SYNTAX = 1382 + ER_WSAS_FAILED = 1383 + ER_DIFF_GROUPS_PROC = 1384 + ER_NO_GROUP_FOR_PROC = 1385 + ER_ORDER_WITH_PROC = 1386 + ER_LOGGING_PROHIBIT_CHANGING_OF = 1387 + ER_NO_FILE_MAPPING = 1388 + ER_WRONG_MAGIC = 1389 + ER_PS_MANY_PARAM = 1390 + ER_KEY_PART_0 = 1391 + ER_VIEW_CHECKSUM = 1392 + ER_VIEW_MULTIUPDATE = 1393 + ER_VIEW_NO_INSERT_FIELD_LIST = 1394 + ER_VIEW_DELETE_MERGE_VIEW = 1395 + ER_CANNOT_USER = 1396 + ER_XAER_NOTA = 1397 + ER_XAER_INVAL = 1398 + ER_XAER_RMFAIL = 1399 + ER_XAER_OUTSIDE = 1400 + ER_XAER_RMERR = 1401 + ER_XA_RBROLLBACK = 1402 + ER_NONEXISTING_PROC_GRANT = 1403 + ER_PROC_AUTO_GRANT_FAIL = 1404 + ER_PROC_AUTO_REVOKE_FAIL = 1405 + ER_DATA_TOO_LONG = 1406 + ER_SP_BAD_SQLSTATE = 1407 + ER_STARTUP = 1408 + ER_LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR = 1409 + ER_CANT_CREATE_USER_WITH_GRANT = 1410 + ER_WRONG_VALUE_FOR_TYPE = 1411 + ER_TABLE_DEF_CHANGED = 1412 + ER_SP_DUP_HANDLER = 1413 + ER_SP_NOT_VAR_ARG = 1414 + ER_SP_NO_RETSET = 1415 + ER_CANT_CREATE_GEOMETRY_OBJECT = 1416 + ER_FAILED_ROUTINE_BREAK_BINLOG = 1417 + ER_BINLOG_UNSAFE_ROUTINE = 1418 + ER_BINLOG_CREATE_ROUTINE_NEED_SUPER = 1419 + ER_EXEC_STMT_WITH_OPEN_CURSOR = 1420 + ER_STMT_HAS_NO_OPEN_CURSOR = 1421 + ER_COMMIT_NOT_ALLOWED_IN_SF_OR_TRG = 1422 + ER_NO_DEFAULT_FOR_VIEW_FIELD = 1423 + ER_SP_NO_RECURSION = 1424 + ER_TOO_BIG_SCALE = 1425 + ER_TOO_BIG_PRECISION = 1426 + ER_M_BIGGER_THAN_D = 1427 + ER_WRONG_LOCK_OF_SYSTEM_TABLE = 1428 + ER_CONNECT_TO_FOREIGN_DATA_SOURCE = 1429 + ER_QUERY_ON_FOREIGN_DATA_SOURCE = 1430 + ER_FOREIGN_DATA_SOURCE_DOESNT_EXIST = 1431 + ER_FOREIGN_DATA_STRING_INVALID_CANT_CREATE = 1432 + ER_FOREIGN_DATA_STRING_INVALID = 1433 + ER_CANT_CREATE_FEDERATED_TABLE = 1434 + ER_TRG_IN_WRONG_SCHEMA = 1435 + ER_STACK_OVERRUN_NEED_MORE = 1436 + ER_TOO_LONG_BODY = 1437 + ER_WARN_CANT_DROP_DEFAULT_KEYCACHE = 1438 + ER_TOO_BIG_DISPLAYWIDTH = 1439 + ER_XAER_DUPID = 1440 + ER_DATETIME_FUNCTION_OVERFLOW = 1441 + ER_CANT_UPDATE_USED_TABLE_IN_SF_OR_TRG = 1442 + ER_VIEW_PREVENT_UPDATE = 1443 + ER_PS_NO_RECURSION = 1444 + ER_SP_CANT_SET_AUTOCOMMIT = 1445 + ER_MALFORMED_DEFINER = 1446 + ER_VIEW_FRM_NO_USER = 1447 + ER_VIEW_OTHER_USER = 1448 + ER_NO_SUCH_USER = 1449 + ER_FORBID_SCHEMA_CHANGE = 1450 + ER_ROW_IS_REFERENCED_2 = 1451 + ER_NO_REFERENCED_ROW_2 = 1452 + ER_SP_BAD_VAR_SHADOW = 1453 + ER_TRG_NO_DEFINER = 1454 + ER_OLD_FILE_FORMAT = 1455 + ER_SP_RECURSION_LIMIT = 1456 + ER_SP_PROC_TABLE_CORRUPT = 1457 + ER_SP_WRONG_NAME = 1458 + ER_TABLE_NEEDS_UPGRADE = 1459 + ER_SP_NO_AGGREGATE = 1460 + ER_MAX_PREPARED_STMT_COUNT_REACHED = 1461 + ER_VIEW_RECURSIVE = 1462 + ER_NON_GROUPING_FIELD_USED = 1463 + ER_TABLE_CANT_HANDLE_SPKEYS = 1464 + ER_NO_TRIGGERS_ON_SYSTEM_SCHEMA = 1465 + ER_REMOVED_SPACES = 1466 + ER_AUTOINC_READ_FAILED = 1467 + ER_USERNAME = 1468 + ER_HOSTNAME = 1469 + ER_WRONG_STRING_LENGTH = 1470 + ER_NON_INSERTABLE_TABLE = 1471 + ER_ADMIN_WRONG_MRG_TABLE = 1472 + ER_TOO_HIGH_LEVEL_OF_NESTING_FOR_SELECT = 1473 + ER_NAME_BECOMES_EMPTY = 1474 + ER_AMBIGUOUS_FIELD_TERM = 1475 + ER_FOREIGN_SERVER_EXISTS = 1476 + ER_FOREIGN_SERVER_DOESNT_EXIST = 1477 + ER_ILLEGAL_HA_CREATE_OPTION = 1478 + ER_PARTITION_REQUIRES_VALUES_ERROR = 1479 + ER_PARTITION_WRONG_VALUES_ERROR = 1480 + ER_PARTITION_MAXVALUE_ERROR = 1481 + ER_PARTITION_SUBPARTITION_ERROR = 1482 + ER_PARTITION_SUBPART_MIX_ERROR = 1483 + ER_PARTITION_WRONG_NO_PART_ERROR = 1484 + ER_PARTITION_WRONG_NO_SUBPART_ERROR = 1485 + ER_WRONG_EXPR_IN_PARTITION_FUNC_ERROR = 1486 + ER_NO_CONST_EXPR_IN_RANGE_OR_LIST_ERROR = 1487 + ER_FIELD_NOT_FOUND_PART_ERROR = 1488 + ER_LIST_OF_FIELDS_ONLY_IN_HASH_ERROR = 1489 + ER_INCONSISTENT_PARTITION_INFO_ERROR = 1490 + ER_PARTITION_FUNC_NOT_ALLOWED_ERROR = 1491 + ER_PARTITIONS_MUST_BE_DEFINED_ERROR = 1492 + ER_RANGE_NOT_INCREASING_ERROR = 1493 + ER_INCONSISTENT_TYPE_OF_FUNCTIONS_ERROR = 1494 + ER_MULTIPLE_DEF_CONST_IN_LIST_PART_ERROR = 1495 + ER_PARTITION_ENTRY_ERROR = 1496 + ER_MIX_HANDLER_ERROR = 1497 + ER_PARTITION_NOT_DEFINED_ERROR = 1498 + ER_TOO_MANY_PARTITIONS_ERROR = 1499 + ER_SUBPARTITION_ERROR = 1500 + ER_CANT_CREATE_HANDLER_FILE = 1501 + ER_BLOB_FIELD_IN_PART_FUNC_ERROR = 1502 + ER_UNIQUE_KEY_NEED_ALL_FIELDS_IN_PF = 1503 + ER_NO_PARTS_ERROR = 1504 + ER_PARTITION_MGMT_ON_NONPARTITIONED = 1505 + ER_FOREIGN_KEY_ON_PARTITIONED = 1506 + ER_DROP_PARTITION_NON_EXISTENT = 1507 + ER_DROP_LAST_PARTITION = 1508 + ER_COALESCE_ONLY_ON_HASH_PARTITION = 1509 + ER_REORG_HASH_ONLY_ON_SAME_NO = 1510 + ER_REORG_NO_PARAM_ERROR = 1511 + ER_ONLY_ON_RANGE_LIST_PARTITION = 1512 + ER_ADD_PARTITION_SUBPART_ERROR = 1513 + ER_ADD_PARTITION_NO_NEW_PARTITION = 1514 + ER_COALESCE_PARTITION_NO_PARTITION = 1515 + ER_REORG_PARTITION_NOT_EXIST = 1516 + ER_SAME_NAME_PARTITION = 1517 + ER_NO_BINLOG_ERROR = 1518 + ER_CONSECUTIVE_REORG_PARTITIONS = 1519 + ER_REORG_OUTSIDE_RANGE = 1520 + ER_PARTITION_FUNCTION_FAILURE = 1521 + ER_PART_STATE_ERROR = 1522 + ER_LIMITED_PART_RANGE = 1523 + ER_PLUGIN_IS_NOT_LOADED = 1524 + ER_WRONG_VALUE = 1525 + ER_NO_PARTITION_FOR_GIVEN_VALUE = 1526 + ER_FILEGROUP_OPTION_ONLY_ONCE = 1527 + ER_CREATE_FILEGROUP_FAILED = 1528 + ER_DROP_FILEGROUP_FAILED = 1529 + ER_TABLESPACE_AUTO_EXTEND_ERROR = 1530 + ER_WRONG_SIZE_NUMBER = 1531 + ER_SIZE_OVERFLOW_ERROR = 1532 + ER_ALTER_FILEGROUP_FAILED = 1533 + ER_BINLOG_ROW_LOGGING_FAILED = 1534 + ER_BINLOG_ROW_WRONG_TABLE_DEF = 1535 + ER_BINLOG_ROW_RBR_TO_SBR = 1536 + ER_EVENT_ALREADY_EXISTS = 1537 + ER_EVENT_STORE_FAILED = 1538 + ER_EVENT_DOES_NOT_EXIST = 1539 + ER_EVENT_CANT_ALTER = 1540 + ER_EVENT_DROP_FAILED = 1541 + ER_EVENT_INTERVAL_NOT_POSITIVE_OR_TOO_BIG = 1542 + ER_EVENT_ENDS_BEFORE_STARTS = 1543 + ER_EVENT_EXEC_TIME_IN_THE_PAST = 1544 + ER_EVENT_OPEN_TABLE_FAILED = 1545 + ER_EVENT_NEITHER_M_EXPR_NOR_M_AT = 1546 + ER_OBSOLETE_COL_COUNT_DOESNT_MATCH_CORRUPTED = 1547 + ER_OBSOLETE_CANNOT_LOAD_FROM_TABLE = 1548 + ER_EVENT_CANNOT_DELETE = 1549 + ER_EVENT_COMPILE_ERROR = 1550 + ER_EVENT_SAME_NAME = 1551 + ER_EVENT_DATA_TOO_LONG = 1552 + ER_DROP_INDEX_FK = 1553 + ER_WARN_DEPRECATED_SYNTAX_WITH_VER = 1554 + ER_CANT_WRITE_LOCK_LOG_TABLE = 1555 + ER_CANT_LOCK_LOG_TABLE = 1556 + ER_FOREIGN_DUPLICATE_KEY_OLD_UNUSED = 1557 + ER_COL_COUNT_DOESNT_MATCH_PLEASE_UPDATE = 1558 + ER_TEMP_TABLE_PREVENTS_SWITCH_OUT_OF_RBR = 1559 + ER_STORED_FUNCTION_PREVENTS_SWITCH_BINLOG_FORMAT = 1560 + ER_NDB_CANT_SWITCH_BINLOG_FORMAT = 1561 + ER_PARTITION_NO_TEMPORARY = 1562 + ER_PARTITION_CONST_DOMAIN_ERROR = 1563 + ER_PARTITION_FUNCTION_IS_NOT_ALLOWED = 1564 + ER_DDL_LOG_ERROR = 1565 + ER_NULL_IN_VALUES_LESS_THAN = 1566 + ER_WRONG_PARTITION_NAME = 1567 + ER_CANT_CHANGE_TX_CHARACTERISTICS = 1568 + ER_DUP_ENTRY_AUTOINCREMENT_CASE = 1569 + ER_EVENT_MODIFY_QUEUE_ERROR = 1570 + ER_EVENT_SET_VAR_ERROR = 1571 + ER_PARTITION_MERGE_ERROR = 1572 + ER_CANT_ACTIVATE_LOG = 1573 + ER_RBR_NOT_AVAILABLE = 1574 + ER_BASE64_DECODE_ERROR = 1575 + ER_EVENT_RECURSION_FORBIDDEN = 1576 + ER_EVENTS_DB_ERROR = 1577 + ER_ONLY_INTEGERS_ALLOWED = 1578 + ER_UNSUPORTED_LOG_ENGINE = 1579 + ER_BAD_LOG_STATEMENT = 1580 + ER_CANT_RENAME_LOG_TABLE = 1581 + ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT = 1582 + ER_WRONG_PARAMETERS_TO_NATIVE_FCT = 1583 + ER_WRONG_PARAMETERS_TO_STORED_FCT = 1584 + ER_NATIVE_FCT_NAME_COLLISION = 1585 + ER_DUP_ENTRY_WITH_KEY_NAME = 1586 + ER_BINLOG_PURGE_EMFILE = 1587 + ER_EVENT_CANNOT_CREATE_IN_THE_PAST = 1588 + ER_EVENT_CANNOT_ALTER_IN_THE_PAST = 1589 + ER_SLAVE_INCIDENT = 1590 + ER_NO_PARTITION_FOR_GIVEN_VALUE_SILENT = 1591 + ER_BINLOG_UNSAFE_STATEMENT = 1592 + ER_SLAVE_FATAL_ERROR = 1593 + ER_SLAVE_RELAY_LOG_READ_FAILURE = 1594 + ER_SLAVE_RELAY_LOG_WRITE_FAILURE = 1595 + ER_SLAVE_CREATE_EVENT_FAILURE = 1596 + ER_SLAVE_MASTER_COM_FAILURE = 1597 + ER_BINLOG_LOGGING_IMPOSSIBLE = 1598 + ER_VIEW_NO_CREATION_CTX = 1599 + ER_VIEW_INVALID_CREATION_CTX = 1600 + ER_SR_INVALID_CREATION_CTX = 1601 + ER_TRG_CORRUPTED_FILE = 1602 + ER_TRG_NO_CREATION_CTX = 1603 + ER_TRG_INVALID_CREATION_CTX = 1604 + ER_EVENT_INVALID_CREATION_CTX = 1605 + ER_TRG_CANT_OPEN_TABLE = 1606 + ER_CANT_CREATE_SROUTINE = 1607 + ER_NEVER_USED = 1608 + ER_NO_FORMAT_DESCRIPTION_EVENT_BEFORE_BINLOG_STATEMENT = 1609 + ER_SLAVE_CORRUPT_EVENT = 1610 + ER_LOAD_DATA_INVALID_COLUMN_UNUSED = 1611 + ER_LOG_PURGE_NO_FILE = 1612 + ER_XA_RBTIMEOUT = 1613 + ER_XA_RBDEADLOCK = 1614 + ER_NEED_REPREPARE = 1615 + ER_DELAYED_NOT_SUPPORTED = 1616 + WARN_NO_MASTER_INFO = 1617 + WARN_OPTION_IGNORED = 1618 + ER_PLUGIN_DELETE_BUILTIN = 1619 + WARN_PLUGIN_BUSY = 1620 + ER_VARIABLE_IS_READONLY = 1621 + ER_WARN_ENGINE_TRANSACTION_ROLLBACK = 1622 + ER_SLAVE_HEARTBEAT_FAILURE = 1623 + ER_SLAVE_HEARTBEAT_VALUE_OUT_OF_RANGE = 1624 + ER_NDB_REPLICATION_SCHEMA_ERROR = 1625 + ER_CONFLICT_FN_PARSE_ERROR = 1626 + ER_EXCEPTIONS_WRITE_ERROR = 1627 + ER_TOO_LONG_TABLE_COMMENT = 1628 + ER_TOO_LONG_FIELD_COMMENT = 1629 + ER_FUNC_INEXISTENT_NAME_COLLISION = 1630 + ER_DATABASE_NAME = 1631 + ER_TABLE_NAME = 1632 + ER_PARTITION_NAME = 1633 + ER_SUBPARTITION_NAME = 1634 + ER_TEMPORARY_NAME = 1635 + ER_RENAMED_NAME = 1636 + ER_TOO_MANY_CONCURRENT_TRXS = 1637 + WARN_NON_ASCII_SEPARATOR_NOT_IMPLEMENTED = 1638 + ER_DEBUG_SYNC_TIMEOUT = 1639 + ER_DEBUG_SYNC_HIT_LIMIT = 1640 + ER_DUP_SIGNAL_SET = 1641 + ER_SIGNAL_WARN = 1642 + ER_SIGNAL_NOT_FOUND = 1643 + ER_SIGNAL_EXCEPTION = 1644 + ER_RESIGNAL_WITHOUT_ACTIVE_HANDLER = 1645 + ER_SIGNAL_BAD_CONDITION_TYPE = 1646 + WARN_COND_ITEM_TRUNCATED = 1647 + ER_COND_ITEM_TOO_LONG = 1648 + ER_UNKNOWN_LOCALE = 1649 + ER_SLAVE_IGNORE_SERVER_IDS = 1650 + ER_QUERY_CACHE_DISABLED = 1651 + ER_SAME_NAME_PARTITION_FIELD = 1652 + ER_PARTITION_COLUMN_LIST_ERROR = 1653 + ER_WRONG_TYPE_COLUMN_VALUE_ERROR = 1654 + ER_TOO_MANY_PARTITION_FUNC_FIELDS_ERROR = 1655 + ER_MAXVALUE_IN_VALUES_IN = 1656 + ER_TOO_MANY_VALUES_ERROR = 1657 + ER_ROW_SINGLE_PARTITION_FIELD_ERROR = 1658 + ER_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD = 1659 + ER_PARTITION_FIELDS_TOO_LONG = 1660 + ER_BINLOG_ROW_ENGINE_AND_STMT_ENGINE = 1661 + ER_BINLOG_ROW_MODE_AND_STMT_ENGINE = 1662 + ER_BINLOG_UNSAFE_AND_STMT_ENGINE = 1663 + ER_BINLOG_ROW_INJECTION_AND_STMT_ENGINE = 1664 + ER_BINLOG_STMT_MODE_AND_ROW_ENGINE = 1665 + ER_BINLOG_ROW_INJECTION_AND_STMT_MODE = 1666 + ER_BINLOG_MULTIPLE_ENGINES_AND_SELF_LOGGING_ENGINE = 1667 + ER_BINLOG_UNSAFE_LIMIT = 1668 + ER_UNUSED4 = 1669 + ER_BINLOG_UNSAFE_SYSTEM_TABLE = 1670 + ER_BINLOG_UNSAFE_AUTOINC_COLUMNS = 1671 + ER_BINLOG_UNSAFE_UDF = 1672 + ER_BINLOG_UNSAFE_SYSTEM_VARIABLE = 1673 + ER_BINLOG_UNSAFE_SYSTEM_FUNCTION = 1674 + ER_BINLOG_UNSAFE_NONTRANS_AFTER_TRANS = 1675 + ER_MESSAGE_AND_STATEMENT = 1676 + ER_SLAVE_CONVERSION_FAILED = 1677 + ER_SLAVE_CANT_CREATE_CONVERSION = 1678 + ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_BINLOG_FORMAT = 1679 + ER_PATH_LENGTH = 1680 + ER_WARN_DEPRECATED_SYNTAX_NO_REPLACEMENT = 1681 + ER_WRONG_NATIVE_TABLE_STRUCTURE = 1682 + ER_WRONG_PERFSCHEMA_USAGE = 1683 + ER_WARN_I_S_SKIPPED_TABLE = 1684 + ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_BINLOG_DIRECT = 1685 + ER_STORED_FUNCTION_PREVENTS_SWITCH_BINLOG_DIRECT = 1686 + ER_SPATIAL_MUST_HAVE_GEOM_COL = 1687 + ER_TOO_LONG_INDEX_COMMENT = 1688 + ER_LOCK_ABORTED = 1689 + ER_DATA_OUT_OF_RANGE = 1690 + ER_WRONG_SPVAR_TYPE_IN_LIMIT = 1691 + ER_BINLOG_UNSAFE_MULTIPLE_ENGINES_AND_SELF_LOGGING_ENGINE = 1692 + ER_BINLOG_UNSAFE_MIXED_STATEMENT = 1693 + ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SQL_LOG_BIN = 1694 + ER_STORED_FUNCTION_PREVENTS_SWITCH_SQL_LOG_BIN = 1695 + ER_FAILED_READ_FROM_PAR_FILE = 1696 + ER_VALUES_IS_NOT_INT_TYPE_ERROR = 1697 + ER_ACCESS_DENIED_NO_PASSWORD_ERROR = 1698 + ER_SET_PASSWORD_AUTH_PLUGIN = 1699 + ER_GRANT_PLUGIN_USER_EXISTS = 1700 + ER_TRUNCATE_ILLEGAL_FK = 1701 + ER_PLUGIN_IS_PERMANENT = 1702 + ER_SLAVE_HEARTBEAT_VALUE_OUT_OF_RANGE_MIN = 1703 + ER_SLAVE_HEARTBEAT_VALUE_OUT_OF_RANGE_MAX = 1704 + ER_STMT_CACHE_FULL = 1705 + ER_MULTI_UPDATE_KEY_CONFLICT = 1706 + ER_TABLE_NEEDS_REBUILD = 1707 + WARN_OPTION_BELOW_LIMIT = 1708 + ER_INDEX_COLUMN_TOO_LONG = 1709 + ER_ERROR_IN_TRIGGER_BODY = 1710 + ER_ERROR_IN_UNKNOWN_TRIGGER_BODY = 1711 + ER_INDEX_CORRUPT = 1712 + ER_UNDO_RECORD_TOO_BIG = 1713 + ER_BINLOG_UNSAFE_INSERT_IGNORE_SELECT = 1714 + ER_BINLOG_UNSAFE_INSERT_SELECT_UPDATE = 1715 + ER_BINLOG_UNSAFE_REPLACE_SELECT = 1716 + ER_BINLOG_UNSAFE_CREATE_IGNORE_SELECT = 1717 + ER_BINLOG_UNSAFE_CREATE_REPLACE_SELECT = 1718 + ER_BINLOG_UNSAFE_UPDATE_IGNORE = 1719 + ER_PLUGIN_NO_UNINSTALL = 1720 + ER_PLUGIN_NO_INSTALL = 1721 + ER_BINLOG_UNSAFE_WRITE_AUTOINC_SELECT = 1722 + ER_BINLOG_UNSAFE_CREATE_SELECT_AUTOINC = 1723 + ER_BINLOG_UNSAFE_INSERT_TWO_KEYS = 1724 + ER_TABLE_IN_FK_CHECK = 1725 + ER_UNSUPPORTED_ENGINE = 1726 + ER_BINLOG_UNSAFE_AUTOINC_NOT_FIRST = 1727 + ER_CANNOT_LOAD_FROM_TABLE_V2 = 1728 + ER_MASTER_DELAY_VALUE_OUT_OF_RANGE = 1729 + ER_ONLY_FD_AND_RBR_EVENTS_ALLOWED_IN_BINLOG_STATEMENT = 1730 + ER_PARTITION_EXCHANGE_DIFFERENT_OPTION = 1731 + ER_PARTITION_EXCHANGE_PART_TABLE = 1732 + ER_PARTITION_EXCHANGE_TEMP_TABLE = 1733 + ER_PARTITION_INSTEAD_OF_SUBPARTITION = 1734 + ER_UNKNOWN_PARTITION = 1735 + ER_TABLES_DIFFERENT_METADATA = 1736 + ER_ROW_DOES_NOT_MATCH_PARTITION = 1737 + ER_BINLOG_CACHE_SIZE_GREATER_THAN_MAX = 1738 + ER_WARN_INDEX_NOT_APPLICABLE = 1739 + ER_PARTITION_EXCHANGE_FOREIGN_KEY = 1740 + ER_NO_SUCH_KEY_VALUE = 1741 + ER_RPL_INFO_DATA_TOO_LONG = 1742 + ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE = 1743 + ER_BINLOG_READ_EVENT_CHECKSUM_FAILURE = 1744 + ER_BINLOG_STMT_CACHE_SIZE_GREATER_THAN_MAX = 1745 + ER_CANT_UPDATE_TABLE_IN_CREATE_TABLE_SELECT = 1746 + ER_PARTITION_CLAUSE_ON_NONPARTITIONED = 1747 + ER_ROW_DOES_NOT_MATCH_GIVEN_PARTITION_SET = 1748 + ER_NO_SUCH_PARTITION__UNUSED = 1749 + ER_CHANGE_RPL_INFO_REPOSITORY_FAILURE = 1750 + ER_WARNING_NOT_COMPLETE_ROLLBACK_WITH_CREATED_TEMP_TABLE = 1751 + ER_WARNING_NOT_COMPLETE_ROLLBACK_WITH_DROPPED_TEMP_TABLE = 1752 + ER_MTS_FEATURE_IS_NOT_SUPPORTED = 1753 + ER_MTS_UPDATED_DBS_GREATER_MAX = 1754 + ER_MTS_CANT_PARALLEL = 1755 + ER_MTS_INCONSISTENT_DATA = 1756 + ER_FULLTEXT_NOT_SUPPORTED_WITH_PARTITIONING = 1757 + ER_DA_INVALID_CONDITION_NUMBER = 1758 + ER_INSECURE_PLAIN_TEXT = 1759 + ER_INSECURE_CHANGE_MASTER = 1760 + ER_FOREIGN_DUPLICATE_KEY_WITH_CHILD_INFO = 1761 + ER_FOREIGN_DUPLICATE_KEY_WITHOUT_CHILD_INFO = 1762 + ER_SQLTHREAD_WITH_SECURE_SLAVE = 1763 + ER_TABLE_HAS_NO_FT = 1764 + ER_VARIABLE_NOT_SETTABLE_IN_SF_OR_TRIGGER = 1765 + ER_VARIABLE_NOT_SETTABLE_IN_TRANSACTION = 1766 + ER_GTID_NEXT_IS_NOT_IN_GTID_NEXT_LIST = 1767 + ER_CANT_CHANGE_GTID_NEXT_IN_TRANSACTION = 1768 + ER_SET_STATEMENT_CANNOT_INVOKE_FUNCTION = 1769 + ER_GTID_NEXT_CANT_BE_AUTOMATIC_IF_GTID_NEXT_LIST_IS_NON_NULL = 1770 + ER_SKIPPING_LOGGED_TRANSACTION = 1771 + ER_MALFORMED_GTID_SET_SPECIFICATION = 1772 + ER_MALFORMED_GTID_SET_ENCODING = 1773 + ER_MALFORMED_GTID_SPECIFICATION = 1774 + ER_GNO_EXHAUSTED = 1775 + ER_BAD_SLAVE_AUTO_POSITION = 1776 + ER_AUTO_POSITION_REQUIRES_GTID_MODE_NOT_OFF = 1777 + ER_CANT_DO_IMPLICIT_COMMIT_IN_TRX_WHEN_GTID_NEXT_IS_SET = 1778 + ER_GTID_MODE_ON_REQUIRES_ENFORCE_GTID_CONSISTENCY_ON = 1779 + ER_GTID_MODE_REQUIRES_BINLOG = 1780 + ER_CANT_SET_GTID_NEXT_TO_GTID_WHEN_GTID_MODE_IS_OFF = 1781 + ER_CANT_SET_GTID_NEXT_TO_ANONYMOUS_WHEN_GTID_MODE_IS_ON = 1782 + ER_CANT_SET_GTID_NEXT_LIST_TO_NON_NULL_WHEN_GTID_MODE_IS_OFF = 1783 + ER_FOUND_GTID_EVENT_WHEN_GTID_MODE_IS_OFF__UNUSED = 1784 + ER_GTID_UNSAFE_NON_TRANSACTIONAL_TABLE = 1785 + ER_GTID_UNSAFE_CREATE_SELECT = 1786 + ER_GTID_UNSAFE_CREATE_DROP_TEMPORARY_TABLE_IN_TRANSACTION = 1787 + ER_GTID_MODE_CAN_ONLY_CHANGE_ONE_STEP_AT_A_TIME = 1788 + ER_MASTER_HAS_PURGED_REQUIRED_GTIDS = 1789 + ER_CANT_SET_GTID_NEXT_WHEN_OWNING_GTID = 1790 + ER_UNKNOWN_EXPLAIN_FORMAT = 1791 + ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION = 1792 + ER_TOO_LONG_TABLE_PARTITION_COMMENT = 1793 + ER_SLAVE_CONFIGURATION = 1794 + ER_INNODB_FT_LIMIT = 1795 + ER_INNODB_NO_FT_TEMP_TABLE = 1796 + ER_INNODB_FT_WRONG_DOCID_COLUMN = 1797 + ER_INNODB_FT_WRONG_DOCID_INDEX = 1798 + ER_INNODB_ONLINE_LOG_TOO_BIG = 1799 + ER_UNKNOWN_ALTER_ALGORITHM = 1800 + ER_UNKNOWN_ALTER_LOCK = 1801 + ER_MTS_CHANGE_MASTER_CANT_RUN_WITH_GAPS = 1802 + ER_MTS_RECOVERY_FAILURE = 1803 + ER_MTS_RESET_WORKERS = 1804 + ER_COL_COUNT_DOESNT_MATCH_CORRUPTED_V2 = 1805 + ER_SLAVE_SILENT_RETRY_TRANSACTION = 1806 + ER_DISCARD_FK_CHECKS_RUNNING = 1807 + ER_TABLE_SCHEMA_MISMATCH = 1808 + ER_TABLE_IN_SYSTEM_TABLESPACE = 1809 + ER_IO_READ_ERROR = 1810 + ER_IO_WRITE_ERROR = 1811 + ER_TABLESPACE_MISSING = 1812 + ER_TABLESPACE_EXISTS = 1813 + ER_TABLESPACE_DISCARDED = 1814 + ER_INTERNAL_ERROR = 1815 + ER_INNODB_IMPORT_ERROR = 1816 + ER_INNODB_INDEX_CORRUPT = 1817 + ER_INVALID_YEAR_COLUMN_LENGTH = 1818 + ER_NOT_VALID_PASSWORD = 1819 + ER_MUST_CHANGE_PASSWORD = 1820 + ER_FK_NO_INDEX_CHILD = 1821 + ER_FK_NO_INDEX_PARENT = 1822 + ER_FK_FAIL_ADD_SYSTEM = 1823 + ER_FK_CANNOT_OPEN_PARENT = 1824 + ER_FK_INCORRECT_OPTION = 1825 + ER_FK_DUP_NAME = 1826 + ER_PASSWORD_FORMAT = 1827 + ER_FK_COLUMN_CANNOT_DROP = 1828 + ER_FK_COLUMN_CANNOT_DROP_CHILD = 1829 + ER_FK_COLUMN_NOT_NULL = 1830 + ER_DUP_INDEX = 1831 + ER_FK_COLUMN_CANNOT_CHANGE = 1832 + ER_FK_COLUMN_CANNOT_CHANGE_CHILD = 1833 + ER_UNUSED5 = 1834 + ER_MALFORMED_PACKET = 1835 + ER_READ_ONLY_MODE = 1836 + ER_GTID_NEXT_TYPE_UNDEFINED_GROUP = 1837 + ER_VARIABLE_NOT_SETTABLE_IN_SP = 1838 + ER_CANT_SET_GTID_PURGED_WHEN_GTID_MODE_IS_OFF = 1839 + ER_CANT_SET_GTID_PURGED_WHEN_GTID_EXECUTED_IS_NOT_EMPTY = 1840 + ER_CANT_SET_GTID_PURGED_WHEN_OWNED_GTIDS_IS_NOT_EMPTY = 1841 + ER_GTID_PURGED_WAS_CHANGED = 1842 + ER_GTID_EXECUTED_WAS_CHANGED = 1843 + ER_BINLOG_STMT_MODE_AND_NO_REPL_TABLES = 1844 + ER_ALTER_OPERATION_NOT_SUPPORTED = 1845 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON = 1846 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_COPY = 1847 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_PARTITION = 1848 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FK_RENAME = 1849 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_COLUMN_TYPE = 1850 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FK_CHECK = 1851 + ER_UNUSED6 = 1852 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_NOPK = 1853 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_AUTOINC = 1854 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_HIDDEN_FTS = 1855 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_CHANGE_FTS = 1856 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FTS = 1857 + ER_SQL_SLAVE_SKIP_COUNTER_NOT_SETTABLE_IN_GTID_MODE = 1858 + ER_DUP_UNKNOWN_IN_INDEX = 1859 + ER_IDENT_CAUSES_TOO_LONG_PATH = 1860 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_NOT_NULL = 1861 + ER_MUST_CHANGE_PASSWORD_LOGIN = 1862 + ER_ROW_IN_WRONG_PARTITION = 1863 + ER_MTS_EVENT_BIGGER_PENDING_JOBS_SIZE_MAX = 1864 + ER_INNODB_NO_FT_USES_PARSER = 1865 + ER_BINLOG_LOGICAL_CORRUPTION = 1866 + ER_WARN_PURGE_LOG_IN_USE = 1867 + ER_WARN_PURGE_LOG_IS_ACTIVE = 1868 + ER_AUTO_INCREMENT_CONFLICT = 1869 + WARN_ON_BLOCKHOLE_IN_RBR = 1870 + ER_SLAVE_MI_INIT_REPOSITORY = 1871 + ER_SLAVE_RLI_INIT_REPOSITORY = 1872 + ER_ACCESS_DENIED_CHANGE_USER_ERROR = 1873 + ER_INNODB_READ_ONLY = 1874 + ER_STOP_SLAVE_SQL_THREAD_TIMEOUT = 1875 + ER_STOP_SLAVE_IO_THREAD_TIMEOUT = 1876 + ER_TABLE_CORRUPT = 1877 + ER_TEMP_FILE_WRITE_FAILURE = 1878 + ER_INNODB_FT_AUX_NOT_HEX_ID = 1879 + ER_OLD_TEMPORALS_UPGRADED = 1880 + ER_INNODB_FORCED_RECOVERY = 1881 + ER_AES_INVALID_IV = 1882 + ER_PLUGIN_CANNOT_BE_UNINSTALLED = 1883 + ER_GTID_UNSAFE_BINLOG_SPLITTABLE_STATEMENT_AND_GTID_GROUP = 1884 + ER_SLAVE_HAS_MORE_GTIDS_THAN_MASTER = 1885 + ER_FILE_CORRUPT = 3000 + ER_ERROR_ON_MASTER = 3001 + ER_INCONSISTENT_ERROR = 3002 + ER_STORAGE_ENGINE_NOT_LOADED = 3003 + ER_GET_STACKED_DA_WITHOUT_ACTIVE_HANDLER = 3004 + ER_WARN_LEGACY_SYNTAX_CONVERTED = 3005 + ER_BINLOG_UNSAFE_FULLTEXT_PLUGIN = 3006 + ER_CANNOT_DISCARD_TEMPORARY_TABLE = 3007 + ER_FK_DEPTH_EXCEEDED = 3008 + ER_COL_COUNT_DOESNT_MATCH_PLEASE_UPDATE_V2 = 3009 + ER_WARN_TRIGGER_DOESNT_HAVE_CREATED = 3010 + ER_REFERENCED_TRG_DOES_NOT_EXIST = 3011 + ER_EXPLAIN_NOT_SUPPORTED = 3012 + ER_INVALID_FIELD_SIZE = 3013 + ER_MISSING_HA_CREATE_OPTION = 3014 + ER_ENGINE_OUT_OF_MEMORY = 3015 + ER_PASSWORD_EXPIRE_ANONYMOUS_USER = 3016 + ER_SLAVE_SQL_THREAD_MUST_STOP = 3017 + ER_NO_FT_MATERIALIZED_SUBQUERY = 3018 + ER_INNODB_UNDO_LOG_FULL = 3019 + ER_INVALID_ARGUMENT_FOR_LOGARITHM = 3020 + ER_SLAVE_CHANNEL_IO_THREAD_MUST_STOP = 3021 + ER_WARN_OPEN_TEMP_TABLES_MUST_BE_ZERO = 3022 + ER_WARN_ONLY_MASTER_LOG_FILE_NO_POS = 3023 + ER_QUERY_TIMEOUT = 3024 + ER_NON_RO_SELECT_DISABLE_TIMER = 3025 + ER_DUP_LIST_ENTRY = 3026 + ER_SQL_MODE_NO_EFFECT = 3027 + ER_AGGREGATE_ORDER_FOR_UNION = 3028 + ER_AGGREGATE_ORDER_NON_AGG_QUERY = 3029 + ER_SLAVE_WORKER_STOPPED_PREVIOUS_THD_ERROR = 3030 + ER_DONT_SUPPORT_SLAVE_PRESERVE_COMMIT_ORDER = 3031 + ER_SERVER_OFFLINE_MODE = 3032 + ER_GIS_DIFFERENT_SRIDS = 3033 + ER_GIS_UNSUPPORTED_ARGUMENT = 3034 + ER_GIS_UNKNOWN_ERROR = 3035 + ER_GIS_UNKNOWN_EXCEPTION = 3036 + ER_GIS_INVALID_DATA = 3037 + ER_BOOST_GEOMETRY_EMPTY_INPUT_EXCEPTION = 3038 + ER_BOOST_GEOMETRY_CENTROID_EXCEPTION = 3039 + ER_BOOST_GEOMETRY_OVERLAY_INVALID_INPUT_EXCEPTION = 3040 + ER_BOOST_GEOMETRY_TURN_INFO_EXCEPTION = 3041 + ER_BOOST_GEOMETRY_SELF_INTERSECTION_POINT_EXCEPTION = 3042 + ER_BOOST_GEOMETRY_UNKNOWN_EXCEPTION = 3043 + ER_STD_BAD_ALLOC_ERROR = 3044 + ER_STD_DOMAIN_ERROR = 3045 + ER_STD_LENGTH_ERROR = 3046 + ER_STD_INVALID_ARGUMENT = 3047 + ER_STD_OUT_OF_RANGE_ERROR = 3048 + ER_STD_OVERFLOW_ERROR = 3049 + ER_STD_RANGE_ERROR = 3050 + ER_STD_UNDERFLOW_ERROR = 3051 + ER_STD_LOGIC_ERROR = 3052 + ER_STD_RUNTIME_ERROR = 3053 + ER_STD_UNKNOWN_EXCEPTION = 3054 + ER_GIS_DATA_WRONG_ENDIANESS = 3055 + ER_CHANGE_MASTER_PASSWORD_LENGTH = 3056 + ER_USER_LOCK_WRONG_NAME = 3057 + ER_USER_LOCK_DEADLOCK = 3058 + ER_REPLACE_INACCESSIBLE_ROWS = 3059 + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_GIS = 3060 + ER_ILLEGAL_USER_VAR = 3061 + ER_GTID_MODE_OFF = 3062 + ER_UNSUPPORTED_BY_REPLICATION_THREAD = 3063 + ER_INCORRECT_TYPE = 3064 + ER_FIELD_IN_ORDER_NOT_SELECT = 3065 + ER_AGGREGATE_IN_ORDER_NOT_SELECT = 3066 + ER_INVALID_RPL_WILD_TABLE_FILTER_PATTERN = 3067 + ER_NET_OK_PACKET_TOO_LARGE = 3068 + ER_INVALID_JSON_DATA = 3069 + ER_INVALID_GEOJSON_MISSING_MEMBER = 3070 + ER_INVALID_GEOJSON_WRONG_TYPE = 3071 + ER_INVALID_GEOJSON_UNSPECIFIED = 3072 + ER_DIMENSION_UNSUPPORTED = 3073 + ER_SLAVE_CHANNEL_DOES_NOT_EXIST = 3074 + ER_SLAVE_MULTIPLE_CHANNELS_HOST_PORT = 3075 + ER_SLAVE_CHANNEL_NAME_INVALID_OR_TOO_LONG = 3076 + ER_SLAVE_NEW_CHANNEL_WRONG_REPOSITORY = 3077 + ER_SLAVE_CHANNEL_DELETE = 3078 + ER_SLAVE_MULTIPLE_CHANNELS_CMD = 3079 + ER_SLAVE_MAX_CHANNELS_EXCEEDED = 3080 + ER_SLAVE_CHANNEL_MUST_STOP = 3081 + ER_SLAVE_CHANNEL_NOT_RUNNING = 3082 + ER_SLAVE_CHANNEL_WAS_RUNNING = 3083 + ER_SLAVE_CHANNEL_WAS_NOT_RUNNING = 3084 + ER_SLAVE_CHANNEL_SQL_THREAD_MUST_STOP = 3085 + ER_SLAVE_CHANNEL_SQL_SKIP_COUNTER = 3086 + ER_WRONG_FIELD_WITH_GROUP_V2 = 3087 + ER_MIX_OF_GROUP_FUNC_AND_FIELDS_V2 = 3088 + ER_WARN_DEPRECATED_SYSVAR_UPDATE = 3089 + ER_WARN_DEPRECATED_SQLMODE = 3090 + ER_CANNOT_LOG_PARTIAL_DROP_DATABASE_WITH_GTID = 3091 + ER_GROUP_REPLICATION_CONFIGURATION = 3092 + ER_GROUP_REPLICATION_RUNNING = 3093 + ER_GROUP_REPLICATION_APPLIER_INIT_ERROR = 3094 + ER_GROUP_REPLICATION_STOP_APPLIER_THREAD_TIMEOUT = 3095 + ER_GROUP_REPLICATION_COMMUNICATION_LAYER_SESSION_ERROR = 3096 + ER_GROUP_REPLICATION_COMMUNICATION_LAYER_JOIN_ERROR = 3097 + ER_BEFORE_DML_VALIDATION_ERROR = 3098 + ER_PREVENTS_VARIABLE_WITHOUT_RBR = 3099 + ER_RUN_HOOK_ERROR = 3100 + ER_TRANSACTION_ROLLBACK_DURING_COMMIT = 3101 + ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED = 3102 + ER_UNSUPPORTED_ALTER_INPLACE_ON_VIRTUAL_COLUMN = 3103 + ER_WRONG_FK_OPTION_FOR_GENERATED_COLUMN = 3104 + ER_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN = 3105 + ER_UNSUPPORTED_ACTION_ON_GENERATED_COLUMN = 3106 + ER_GENERATED_COLUMN_NON_PRIOR = 3107 + ER_DEPENDENT_BY_GENERATED_COLUMN = 3108 + ER_GENERATED_COLUMN_REF_AUTO_INC = 3109 + ER_FEATURE_NOT_AVAILABLE = 3110 + ER_CANT_SET_GTID_MODE = 3111 + ER_CANT_USE_AUTO_POSITION_WITH_GTID_MODE_OFF = 3112 + ER_CANT_REPLICATE_ANONYMOUS_WITH_AUTO_POSITION = 3113 + ER_CANT_REPLICATE_ANONYMOUS_WITH_GTID_MODE_ON = 3114 + ER_CANT_REPLICATE_GTID_WITH_GTID_MODE_OFF = 3115 + ER_CANT_SET_ENFORCE_GTID_CONSISTENCY_ON_WITH_ONGOING_GTID_VIOLATING_TRANSACTIONS = 3116 + ER_SET_ENFORCE_GTID_CONSISTENCY_WARN_WITH_ONGOING_GTID_VIOLATING_TRANSACTIONS = 3117 + ER_ACCOUNT_HAS_BEEN_LOCKED = 3118 + ER_WRONG_TABLESPACE_NAME = 3119 + ER_TABLESPACE_IS_NOT_EMPTY = 3120 + ER_WRONG_FILE_NAME = 3121 + ER_BOOST_GEOMETRY_INCONSISTENT_TURNS_EXCEPTION = 3122 + ER_WARN_OPTIMIZER_HINT_SYNTAX_ERROR = 3123 + ER_WARN_BAD_MAX_EXECUTION_TIME = 3124 + ER_WARN_UNSUPPORTED_MAX_EXECUTION_TIME = 3125 + ER_WARN_CONFLICTING_HINT = 3126 + ER_WARN_UNKNOWN_QB_NAME = 3127 + ER_UNRESOLVED_HINT_NAME = 3128 + ER_WARN_ON_MODIFYING_GTID_EXECUTED_TABLE = 3129 + ER_PLUGGABLE_PROTOCOL_COMMAND_NOT_SUPPORTED = 3130 + ER_LOCKING_SERVICE_WRONG_NAME = 3131 + ER_LOCKING_SERVICE_DEADLOCK = 3132 + ER_LOCKING_SERVICE_TIMEOUT = 3133 + ER_GIS_MAX_POINTS_IN_GEOMETRY_OVERFLOWED = 3134 + ER_SQL_MODE_MERGED = 3135 + ER_VTOKEN_PLUGIN_TOKEN_MISMATCH = 3136 + ER_VTOKEN_PLUGIN_TOKEN_NOT_FOUND = 3137 + ER_CANT_SET_VARIABLE_WHEN_OWNING_GTID = 3138 + ER_SLAVE_CHANNEL_OPERATION_NOT_ALLOWED = 3139 + ER_INVALID_JSON_TEXT = 3140 + ER_INVALID_JSON_TEXT_IN_PARAM = 3141 + ER_INVALID_JSON_BINARY_DATA = 3142 + ER_INVALID_JSON_PATH = 3143 + ER_INVALID_JSON_CHARSET = 3144 + ER_INVALID_JSON_CHARSET_IN_FUNCTION = 3145 + ER_INVALID_TYPE_FOR_JSON = 3146 + ER_INVALID_CAST_TO_JSON = 3147 + ER_INVALID_JSON_PATH_CHARSET = 3148 + ER_INVALID_JSON_PATH_WILDCARD = 3149 + ER_JSON_VALUE_TOO_BIG = 3150 + ER_JSON_KEY_TOO_BIG = 3151 + ER_JSON_USED_AS_KEY = 3152 + ER_JSON_VACUOUS_PATH = 3153 + ER_JSON_BAD_ONE_OR_ALL_ARG = 3154 + ER_NUMERIC_JSON_VALUE_OUT_OF_RANGE = 3155 + ER_INVALID_JSON_VALUE_FOR_CAST = 3156 + ER_JSON_DOCUMENT_TOO_DEEP = 3157 + ER_JSON_DOCUMENT_NULL_KEY = 3158 + ER_SECURE_TRANSPORT_REQUIRED = 3159 + ER_NO_SECURE_TRANSPORTS_CONFIGURED = 3160 + ER_DISABLED_STORAGE_ENGINE = 3161 + ER_USER_DOES_NOT_EXIST = 3162 + ER_USER_ALREADY_EXISTS = 3163 + ER_AUDIT_API_ABORT = 3164 + ER_INVALID_JSON_PATH_ARRAY_CELL = 3165 + ER_BUFPOOL_RESIZE_INPROGRESS = 3166 + ER_FEATURE_DISABLED_SEE_DOC = 3167 + ER_SERVER_ISNT_AVAILABLE = 3168 + ER_SESSION_WAS_KILLED = 3169 + ER_CAPACITY_EXCEEDED = 3170 + ER_CAPACITY_EXCEEDED_IN_RANGE_OPTIMIZER = 3171 + ER_TABLE_NEEDS_UPG_PART = 3172 + ER_CANT_WAIT_FOR_EXECUTED_GTID_SET_WHILE_OWNING_A_GTID = 3173 + ER_CANNOT_ADD_FOREIGN_BASE_COL_VIRTUAL = 3174 + ER_CANNOT_CREATE_VIRTUAL_INDEX_CONSTRAINT = 3175 + ER_ERROR_ON_MODIFYING_GTID_EXECUTED_TABLE = 3176 + ER_LOCK_REFUSED_BY_ENGINE = 3177 + ER_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN = 3178 + ER_MASTER_KEY_ROTATION_NOT_SUPPORTED_BY_SE = 3179 + ER_MASTER_KEY_ROTATION_ERROR_BY_SE = 3180 + ER_MASTER_KEY_ROTATION_BINLOG_FAILED = 3181 + ER_MASTER_KEY_ROTATION_SE_UNAVAILABLE = 3182 + ER_TABLESPACE_CANNOT_ENCRYPT = 3183 + ER_INVALID_ENCRYPTION_OPTION = 3184 + ER_CANNOT_FIND_KEY_IN_KEYRING = 3185 + ER_CAPACITY_EXCEEDED_IN_PARSER = 3186 + ER_UNSUPPORTED_ALTER_ENCRYPTION_INPLACE = 3187 + ER_KEYRING_UDF_KEYRING_SERVICE_ERROR = 3188 + ER_USER_COLUMN_OLD_LENGTH = 3189 +) From 15ce4746396f4ba17068e6a3def8909685c6d4d8 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 27 Sep 2018 14:32:54 +0200 Subject: [PATCH 05/28] wip --- pkg/models/alert_notifications.go | 4 +- pkg/services/alerting/interfaces.go | 17 ++-- pkg/services/alerting/notifier.go | 96 ++++++++++++--------- pkg/services/alerting/notifiers/base.go | 11 +-- pkg/services/sqlstore/alert_notification.go | 32 ++++++- 5 files changed, 98 insertions(+), 62 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index c3c41dc5dd9..b46a09ea345 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -96,9 +96,7 @@ type AlertNotificationState struct { } type SetAlertNotificationStateToPendingCommand struct { - Id int64 - SentAt int64 - Version int64 + State *AlertNotificationState } type SetAlertNotificationStateToCompleteCommand struct { diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 46f8b3c769c..65369ed3884 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -3,6 +3,8 @@ package alerting import ( "context" "time" + + "github.com/grafana/grafana/pkg/models" ) type EvalHandler interface { @@ -20,7 +22,7 @@ type Notifier interface { NeedsImage() bool // ShouldNotify checks this evaluation should send an alert notification - ShouldNotify(ctx context.Context, evalContext *EvalContext) bool + ShouldNotify(ctx context.Context, evalContext *EvalContext, notificationState *models.AlertNotificationState) bool GetNotifierId() int64 GetIsDefault() bool @@ -28,11 +30,16 @@ type Notifier interface { GetFrequency() time.Duration } -type NotifierSlice []Notifier +type NotifierState struct { + notifier Notifier + state *models.AlertNotificationState +} -func (notifiers NotifierSlice) ShouldUploadImage() bool { - for _, notifier := range notifiers { - if notifier.NeedsImage() { +type NotifierStateSlice []*NotifierState + +func (notifiers NotifierStateSlice) ShouldUploadImage() bool { + for _, ns := range notifiers { + if ns.notifier.NeedsImage() { return true } } diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 941d3df9dc3..d9d0e278e99 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1,7 +1,6 @@ package alerting import ( - "context" "errors" "fmt" @@ -39,64 +38,59 @@ type notificationService struct { } func (n *notificationService) SendIfNeeded(context *EvalContext) error { - notifiers, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context) + notifierStates, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context) if err != nil { return err } - if len(notifiers) == 0 { + if len(notifierStates) == 0 { return nil } - if notifiers.ShouldUploadImage() { + if notifierStates.ShouldUploadImage() { if err = n.uploadImage(context); err != nil { n.log.Error("Failed to upload alert panel image.", "error", err) } } - return n.sendNotifications(context, notifiers) + // get alert notification (version = 1) + // phantomjs 15 sek + // loopa notifier - ge mig ett lÃ¥s! where version = 1 + // send notification + // Släpp lÃ¥s + // + + return n.sendNotifications(context, notifierStates) } -func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error { - for _, notifier := range notifiers { - not := notifier +func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *NotifierState) error { + return nil +} - err := bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error { - n.log.Debug("trying to send notification", "id", not.GetNotifierId()) +func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { + n.log.Debug("trying to send notification", "id", notifierState.notifier.GetNotifierId()) - // insert if needed + setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ + State: notifierState.state, + } - // Verify that we can send the notification again - // but this time within the same transaction. - // if !evalContext.IsTestRun && !not.ShouldNotify(ctx, evalContext) { - // return nil - // } + err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd) + if err == m.ErrAlertNotificationStateVersionConflict { + return nil + } - // n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - // metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + if err != nil { + return err + } - // //send notification - // // success := not.Notify(evalContext) == nil - - // if evalContext.IsTestRun { - // return nil - // } - - //write result to db. - // cmd := &m.RecordNotificationJournalCommand{ - // OrgId: evalContext.Rule.OrgId, - // AlertId: evalContext.Rule.Id, - // NotifierId: not.GetNotifierId(), - // SentAt: time.Now().Unix(), - // Success: success, - // } - - // return bus.DispatchCtx(ctx, cmd) - return nil - }) + return n.sendAndMarkAsComplete(evalContext, notifierState) +} +func (n *notificationService) sendNotifications(evalContext *EvalContext, notifierStates NotifierStateSlice) error { + for _, notifierState := range notifierStates { + err := n.sendNotification(evalContext, notifierState) if err != nil { - n.log.Error("failed to send notification", "id", not.GetNotifierId()) + n.log.Error("failed to send notification", "id", notifierState.notifier.GetNotifierId()) } } @@ -143,22 +137,38 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { return nil } -func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierSlice, error) { +func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierStateSlice, error) { query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds} if err := bus.Dispatch(query); err != nil { return nil, err } - var result []Notifier + var result NotifierStateSlice for _, notification := range query.Result { not, err := n.createNotifierFor(notification) if err != nil { - return nil, err + n.log.Error("Could not create notifier", "notifier", notification.Id) + continue } - if not.ShouldNotify(evalContext.Ctx, evalContext) { - result = append(result, not) + query := &m.GetNotificationStateQuery{ + NotifierId: notification.Id, + AlertId: evalContext.Rule.Id, + OrgId: evalContext.Rule.OrgId, + } + + err = bus.DispatchCtx(evalContext.Ctx, query) + if err != nil { + n.log.Error("Could not get notification state.", "notifier", notification.Id) + continue + } + + if not.ShouldNotify(evalContext.Ctx, evalContext, query.Result) { + result = append(result, &NotifierState{ + notifier: not, + state: query.Result, + }) } } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index f71a41235f7..e37ed92aa89 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -48,15 +48,6 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ return false } - // get last successfully sent notification - // lastNotify := time.Time{} - // for _, j := range journals { - // if j.Success { - // lastNotify = time.Unix(j.SentAt, 0) - // break - // } - // } - // Do not notify if interval has not elapsed lastNotify := time.Unix(notificationState.SentAt, 0) if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { @@ -77,7 +68,7 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ } // ShouldNotify checks this evaluation should send an alert notification -func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext) bool { +func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { cmd := &models.GetNotificationStateQuery{ OrgId: c.Rule.OrgId, AlertId: c.Rule.Id, diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index a89e777ede3..403810b772b 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -283,7 +283,7 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl id = ? AND version = ?` - res, err := sess.Exec(sql, m.AlertNotificationStatePending, cmd.Version+1, cmd.Id, cmd.Version) + res, err := sess.Exec(sql, m.AlertNotificationStatePending, cmd.State.Version+1, cmd.State.Id, cmd.State.Version) if err != nil { return err } @@ -308,11 +308,41 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). Get(nj) + // if exists, return it, otherwise create it with default values if err != nil { return err } if !exist { + notificationState := &m.AlertNotificationState{ + OrgId: cmd.OrgId, + AlertId: cmd.AlertId, + NotifierId: cmd.NotifierId, + State: "unknown", + } + + _, err := sess.Insert(notificationState) + + if err == nil { + return nil + } + + uniqenessIndexFailureCodes := []string{ + "UNIQUE constraint failed", + "pq: duplicate key value violates unique constraint", + "Error 1062: Duplicate entry ", + } + + var alreadyExists bool + + for _, code := range uniqenessIndexFailureCodes { + if strings.HasPrefix(err.Error(), code) { + alreadyExists = true + } + } + + return err + return m.ErrAlertNotificationStateNotFound } From 1a75aa54de48ade8412fd4eaf820833e4870c226 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Fri, 28 Sep 2018 10:48:08 +0200 Subject: [PATCH 06/28] wip: impl so that get alertstate also creates it if it does not exist --- .../alerting/notifiers/alertmanager.go | 2 +- pkg/services/alerting/notifiers/base_test.go | 41 ++++---- pkg/services/alerting/test_notification.go | 2 +- pkg/services/sqlstore/alert_notification.go | 41 +++++--- .../sqlstore/alert_notification_test.go | 97 ++++++++++--------- 5 files changed, 97 insertions(+), 86 deletions(-) diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 9826dd1dffb..2caa4d5ab58 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -46,7 +46,7 @@ type AlertmanagerNotifier struct { log log.Logger } -func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext) bool { +func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext, notificationState *m.AlertNotificationState) bool { this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState) // Do not notify when we become OK for the first time. diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index c14006637a4..385acc39f1d 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -2,12 +2,9 @@ package notifiers import ( "context" - "errors" "testing" "time" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" @@ -126,25 +123,27 @@ func TestShouldSendAlertNotification(t *testing.T) { func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { Convey("base notifier", t, func() { - bus.ClearBusHandlers() + //bus.ClearBusHandlers() + // + //notifier := NewNotifierBase(&m.AlertNotification{ + // Id: 1, + // Name: "name", + // Type: "email", + // Settings: simplejson.New(), + //}) + //evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{}) + // + //Convey("should not notify query returns error", func() { + // bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetNotificationStateQuery) error { + // return errors.New("some kind of error unknown error") + // }) + // + // if notifier.ShouldNotify(context.Background(), evalContext) { + // t.Errorf("should not send notifications when query returns error") + // } + //}) - notifier := NewNotifierBase(&m.AlertNotification{ - Id: 1, - Name: "name", - Type: "email", - Settings: simplejson.New(), - }) - evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{}) - - Convey("should not notify query returns error", func() { - bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetNotificationStateQuery) error { - return errors.New("some kind of error unknown error") - }) - - if notifier.ShouldNotify(context.Background(), evalContext) { - t.Errorf("should not send notifications when query returns error") - } - }) + t.Error("might not need this anymore, at least not like this, control flow has changedd") }) } diff --git a/pkg/services/alerting/test_notification.go b/pkg/services/alerting/test_notification.go index 8421360b5ed..228ec90001d 100644 --- a/pkg/services/alerting/test_notification.go +++ b/pkg/services/alerting/test_notification.go @@ -39,7 +39,7 @@ func handleNotificationTestCommand(cmd *NotificationTestCommand) error { return err } - return notifier.sendNotifications(createTestEvalContext(cmd), []Notifier{notifiers}) + return notifier.sendNotifications(createTestEvalContext(cmd), NotifierStateSlice{{notifier: notifiers}}) } func createTestEvalContext(cmd *NotificationTestCommand) *EvalContext { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 403810b772b..7af22016d73 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -302,17 +302,20 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ return withDbSession(ctx, func(sess *DBSession) error { nj := &m.AlertNotificationState{} - exist, err := sess.Desc("alert_notification_state.sent_at"). - Where("alert_notification_state.org_id = ?", cmd.OrgId). - Where("alert_notification_state.alert_id = ?", cmd.AlertId). - Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). - Get(nj) + exist, err := getAlertNotificationState(sess, cmd, nj) // if exists, return it, otherwise create it with default values if err != nil { return err } + if exist { + cmd.Result = nj + return nil + } + + // normally flow ends here + if !exist { notificationState := &m.AlertNotificationState{ OrgId: cmd.OrgId, @@ -323,30 +326,38 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ _, err := sess.Insert(notificationState) - if err == nil { - return nil - } - uniqenessIndexFailureCodes := []string{ "UNIQUE constraint failed", "pq: duplicate key value violates unique constraint", "Error 1062: Duplicate entry ", } - var alreadyExists bool - for _, code := range uniqenessIndexFailureCodes { if strings.HasPrefix(err.Error(), code) { - alreadyExists = true + exist, err = getAlertNotificationState(sess, cmd, nj) + + if exist && err == nil { + cmd.Result = nj + return nil + } } } - return err - - return m.ErrAlertNotificationStateNotFound + if err != nil { + return err + } } cmd.Result = nj return nil }) } + +func getAlertNotificationState(sess *DBSession, cmd *m.GetNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { + exist, err := sess.Desc("alert_notification_state.sent_at"). + Where("alert_notification_state.org_id = ?", cmd.OrgId). + Where("alert_notification_state.alert_id = ?", cmd.AlertId). + Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). + Get(nj) + return exist, err +} diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 206c96b5c6a..52682e7788f 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -1,7 +1,6 @@ package sqlstore import ( - "context" "testing" "time" @@ -14,55 +13,57 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) - Convey("Alert notification state", func() { - var alertId int64 = 7 - var orgId int64 = 5 - var notifierId int64 = 10 + //Convey("Alert notification state", func() { + //var alertId int64 = 7 + //var orgId int64 = 5 + //var notifierId int64 = 10 - Convey("Getting no existant state returns error", func() { - query := &models.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} - err := GetAlertNotificationState(context.Background(), query) - So(err, ShouldEqual, models.ErrAlertNotificationStateNotFound) - }) + //Convey("Getting no existant state returns error", func() { + // query := &models.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} + // err := GetAlertNotificationState(context.Background(), query) + // So(err, ShouldEqual, models.ErrAlertNotificationStateNotFound) + //}) - Convey("Can insert new state for alert notifier", func() { - createCmd := &models.InsertAlertNotificationCommand{ - AlertId: alertId, - NotifierId: notifierId, - OrgId: orgId, - SentAt: 1, - State: models.AlertNotificationStateCompleted, - } - - err := InsertAlertNotificationState(context.Background(), createCmd) - So(err, ShouldBeNil) - - err = InsertAlertNotificationState(context.Background(), createCmd) - So(err, ShouldEqual, models.ErrAlertNotificationStateAlreadyExist) - - Convey("should be able to update alert notifier state", func() { - updateCmd := &models.SetAlertNotificationStateToPendingCommand{ - Id: 1, - SentAt: 1, - Version: 0, - } - - err := SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) - So(err, ShouldBeNil) - - Convey("should not be able to set pending on old version", func() { - err = SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) - So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) - }) - - Convey("should be able to set state to completed", func() { - cmd := &models.SetAlertNotificationStateToCompleteCommand{Id: 1} - err = SetAlertNotificationStateToCompleteCommand(context.Background(), cmd) - So(err, ShouldBeNil) - }) - }) - }) - }) + //Convey("Can insert new state for alert notifier", func() { + // createCmd := &models.InsertAlertNotificationCommand{ + // AlertId: alertId, + // NotifierId: notifierId, + // OrgId: orgId, + // SentAt: 1, + // State: models.AlertNotificationStateCompleted, + // } + // + // err := InsertAlertNotificationState(context.Background(), createCmd) + // So(err, ShouldBeNil) + // + // err = InsertAlertNotificationState(context.Background(), createCmd) + // So(err, ShouldEqual, models.ErrAlertNotificationStateAlreadyExist) + // + // Convey("should be able to update alert notifier state", func() { + // updateCmd := &models.SetAlertNotificationStateToPendingCommand{ + // State: models.AlertNotificationState{ + // Id: 1, + // SentAt: 1, + // Version: 0, + // } + // } + // + // err := SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) + // So(err, ShouldBeNil) + // + // Convey("should not be able to set pending on old version", func() { + // err = SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) + // So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + // }) + // + // Convey("should be able to set state to completed", func() { + // cmd := &models.SetAlertNotificationStateToCompleteCommand{Id: 1} + // err = SetAlertNotificationStateToCompleteCommand(context.Background(), cmd) + // So(err, ShouldBeNil) + // }) + // }) + // }) + //}) Convey("Alert notifications should be empty", func() { cmd := &models.GetAlertNotificationsQuery{ From 88bbc452a7ce5dc502a675ff250072d44b0acf45 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 11:14:36 +0200 Subject: [PATCH 07/28] wip: send and mark as complete --- pkg/models/alert_notifications.go | 5 +++-- pkg/services/alerting/notifier.go | 25 ++++++++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index b46a09ea345..5f7532576c0 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -100,8 +100,9 @@ type SetAlertNotificationStateToPendingCommand struct { } type SetAlertNotificationStateToCompleteCommand struct { - Id int64 - SentAt int64 + Id int64 + Version int64 + SentAt int64 } type GetNotificationStateQuery struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index d9d0e278e99..0de2abc9e0a 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -3,6 +3,7 @@ package alerting import ( "errors" "fmt" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" @@ -53,17 +54,27 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { } } - // get alert notification (version = 1) - // phantomjs 15 sek - // loopa notifier - ge mig ett lÃ¥s! where version = 1 - // send notification - // Släpp lÃ¥s - // - return n.sendNotifications(context, notifierStates) } func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *NotifierState) error { + err := notifierState.notifier.Notify(evalContext) + + cmd := &m.SetAlertNotificationStateToCompleteCommand{ + Id: notifierState.state.Id, + Version: notifierState.state.Version, + SentAt: time.Now().Unix(), + } + + err = bus.DispatchCtx(evalContext.Ctx, cmd) + if err == m.ErrAlertNotificationStateVersionConflict { + return nil + } + + if err != nil { + return err + } + return nil } From 69cc24ea3f76aafb7346333b30074107b86ad9c2 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 11:17:03 +0200 Subject: [PATCH 08/28] wip: test get alert notification state --- pkg/services/sqlstore/alert_notification.go | 66 +++++++++---------- .../sqlstore/alert_notification_test.go | 30 ++++++--- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 7af22016d73..213b715c941 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -3,6 +3,7 @@ package sqlstore import ( "bytes" "context" + "errors" "fmt" "strings" "time" @@ -255,11 +256,12 @@ func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotific func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { return withDbSession(ctx, func(sess *DBSession) error { sql := `UPDATE alert_notification_state SET - state= ? + state = ?, + version = ? WHERE id = ?` - res, err := sess.Exec(sql, m.AlertNotificationStateCompleted, cmd.Id) + res, err := sess.Exec(sql, m.AlertNotificationStateCompleted, cmd.Id, cmd.Version+1) if err != nil { return err } @@ -277,7 +279,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { return withDbSession(ctx, func(sess *DBSession) error { sql := `UPDATE alert_notification_state SET - state= ?, + state = ?, version = ? WHERE id = ? AND @@ -314,41 +316,33 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ return nil } - // normally flow ends here - - if !exist { - notificationState := &m.AlertNotificationState{ - OrgId: cmd.OrgId, - AlertId: cmd.AlertId, - NotifierId: cmd.NotifierId, - State: "unknown", - } - - _, err := sess.Insert(notificationState) - - uniqenessIndexFailureCodes := []string{ - "UNIQUE constraint failed", - "pq: duplicate key value violates unique constraint", - "Error 1062: Duplicate entry ", - } - - for _, code := range uniqenessIndexFailureCodes { - if strings.HasPrefix(err.Error(), code) { - exist, err = getAlertNotificationState(sess, cmd, nj) - - if exist && err == nil { - cmd.Result = nj - return nil - } - } - } - - if err != nil { - return err - } + notificationState := &m.AlertNotificationState{ + OrgId: cmd.OrgId, + AlertId: cmd.AlertId, + NotifierId: cmd.NotifierId, + State: "unknown", } - cmd.Result = nj + if _, err := sess.Insert(notificationState); err != nil { + if dialect.IsUniqueConstraintViolation(err) { + exist, err = getAlertNotificationState(sess, cmd, nj) + + if err != nil { + return err + } + + if !exist { + return errors.New("Should not happen") + } + + cmd.Result = nj + return nil + } + + return err + } + + cmd.Result = notificationState return nil }) } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 52682e7788f..849902359cc 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "testing" "time" @@ -13,16 +14,27 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) - //Convey("Alert notification state", func() { - //var alertId int64 = 7 - //var orgId int64 = 5 - //var notifierId int64 = 10 + Convey("Alert notification state", func() { + var alertID int64 = 7 + var orgID int64 = 5 + var notifierID int64 = 10 - //Convey("Getting no existant state returns error", func() { - // query := &models.GetNotificationStateQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} - // err := GetAlertNotificationState(context.Background(), query) - // So(err, ShouldEqual, models.ErrAlertNotificationStateNotFound) - //}) + Convey("Get no existing state should create a new state", func() { + query := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetAlertNotificationState(context.Background(), query) + So(err, ShouldBeNil) + So(query.Result, ShouldNotBeNil) + So(query.Result.State, ShouldEqual, "unknown") + + Convey("Get existing state should not create a new state", func() { + query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetAlertNotificationState(context.Background(), query2) + So(err, ShouldBeNil) + So(query2.Result, ShouldNotBeNil) + So(query2.Result.Id, ShouldEqual, query.Result.Id) + }) + }) + }) //Convey("Can insert new state for alert notifier", func() { // createCmd := &models.InsertAlertNotificationCommand{ From 2bf399d3c83a3214605efd8f4a4c6dc8ec1126d6 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 12:59:35 +0200 Subject: [PATCH 09/28] No need to get alert notification state in ShouldNotify --- pkg/services/alerting/notifiers/base.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index e37ed92aa89..fa24f925817 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -4,7 +4,6 @@ import ( "context" "time" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" @@ -69,19 +68,7 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ // ShouldNotify checks this evaluation should send an alert notification func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { - cmd := &models.GetNotificationStateQuery{ - OrgId: c.Rule.OrgId, - AlertId: c.Rule.Id, - NotifierId: n.Id, - } - - err := bus.DispatchCtx(ctx, cmd) - if err != nil { - n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err) - return false - } - - return defaultShouldNotify(c, n.SendReminder, n.Frequency, cmd.Result) + return defaultShouldNotify(c, n.SendReminder, n.Frequency, notiferState) } func (n *NotifierBase) GetType() string { From c1763508e0ef492b891ca8005da561022db0626d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 14:12:26 +0200 Subject: [PATCH 10/28] handle pending and completed state for alert notifications --- pkg/models/alert_notifications.go | 4 +- pkg/services/alerting/notifier.go | 51 +++++---- pkg/services/alerting/notifiers/base_test.go | 26 ----- pkg/services/sqlstore/alert_notification.go | 21 +++- .../sqlstore/alert_notification_test.go | 100 +++++++++++------- 5 files changed, 108 insertions(+), 94 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 5f7532576c0..14bf8694207 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -100,9 +100,7 @@ type SetAlertNotificationStateToPendingCommand struct { } type SetAlertNotificationStateToCompleteCommand struct { - Id int64 - Version int64 - SentAt int64 + State *AlertNotificationState } type GetNotificationStateQuery struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 0de2abc9e0a..a80fb265e81 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -58,20 +59,32 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { } func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *NotifierState) error { - err := notifierState.notifier.Notify(evalContext) + not := notifierState.notifier + n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) + metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - cmd := &m.SetAlertNotificationStateToCompleteCommand{ - Id: notifierState.state.Id, - Version: notifierState.state.Version, - SentAt: time.Now().Unix(), + err := not.Notify(evalContext) + + if err != nil { + n.log.Error("failed to send notification", "id", not.GetNotifierId()) + } else { + notifierState.state.SentAt = time.Now().Unix() } - err = bus.DispatchCtx(evalContext.Ctx, cmd) - if err == m.ErrAlertNotificationStateVersionConflict { + if evalContext.IsTestRun { return nil } - if err != nil { + cmd := &m.SetAlertNotificationStateToCompleteCommand{ + State: notifierState.state, + } + + if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { + if err == m.ErrAlertNotificationStateVersionConflict { + n.log.Error("notification state out of sync", "id", not.GetNotifierId()) + return nil + } + return err } @@ -79,19 +92,19 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no } func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { - n.log.Debug("trying to send notification", "id", notifierState.notifier.GetNotifierId()) + if !evalContext.IsTestRun { + setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ + State: notifierState.state, + } - setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ - State: notifierState.state, - } + err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd) + if err == m.ErrAlertNotificationStateVersionConflict { + return nil + } - err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd) - if err == m.ErrAlertNotificationStateVersionConflict { - return nil - } - - if err != nil { - return err + if err != nil { + return err + } } return n.sendAndMarkAsComplete(evalContext, notifierState) diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 385acc39f1d..50cfbef7387 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -121,32 +121,6 @@ func TestShouldSendAlertNotification(t *testing.T) { } } -func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { - Convey("base notifier", t, func() { - //bus.ClearBusHandlers() - // - //notifier := NewNotifierBase(&m.AlertNotification{ - // Id: 1, - // Name: "name", - // Type: "email", - // Settings: simplejson.New(), - //}) - //evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{}) - // - //Convey("should not notify query returns error", func() { - // bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetNotificationStateQuery) error { - // return errors.New("some kind of error unknown error") - // }) - // - // if notifier.ShouldNotify(context.Background(), evalContext) { - // t.Errorf("should not send notifications when query returns error") - // } - //}) - - t.Error("might not need this anymore, at least not like this, control flow has changedd") - }) -} - func TestBaseNotifier(t *testing.T) { Convey("default constructor for notifiers", t, func() { bJson := simplejson.New() diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 213b715c941..fdf467695dc 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -255,20 +255,26 @@ func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotific func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { return withDbSession(ctx, func(sess *DBSession) error { + version := cmd.State.Version + var current m.AlertNotificationState + sess.ID(cmd.State.Id).Get(¤t) + + cmd.State.State = m.AlertNotificationStateCompleted + cmd.State.Version++ + sql := `UPDATE alert_notification_state SET state = ?, version = ? WHERE id = ?` - res, err := sess.Exec(sql, m.AlertNotificationStateCompleted, cmd.Id, cmd.Version+1) + _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.Id) + if err != nil { return err } - affected, _ := res.RowsAffected() - - if affected == 0 { + if current.Version != version { return m.ErrAlertNotificationStateVersionConflict } @@ -278,6 +284,10 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { return withDbSession(ctx, func(sess *DBSession) error { + currentVersion := cmd.State.Version + cmd.State.State = m.AlertNotificationStatePending + cmd.State.Version++ + sql := `UPDATE alert_notification_state SET state = ?, version = ? @@ -285,7 +295,8 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl id = ? AND version = ?` - res, err := sess.Exec(sql, m.AlertNotificationStatePending, cmd.State.Version+1, cmd.State.Id, cmd.State.Version) + res, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.Id, currentVersion) + if err != nil { return err } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 849902359cc..daed5a8cd7f 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -25,6 +25,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) So(query.Result, ShouldNotBeNil) So(query.Result.State, ShouldEqual, "unknown") + So(query.Result.Version, ShouldEqual, 0) Convey("Get existing state should not create a new state", func() { query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} @@ -33,50 +34,67 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(query2.Result, ShouldNotBeNil) So(query2.Result.Id, ShouldEqual, query.Result.Id) }) + + Convey("Update existing state to pending with correct version should update database", func() { + s := *query.Result + cmd := models.SetAlertNotificationStateToPendingCommand{ + State: &s, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + So(cmd.State.Version, ShouldEqual, 1) + So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending) + + query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetAlertNotificationState(context.Background(), query2) + So(err, ShouldBeNil) + So(query2.Result.Version, ShouldEqual, 1) + So(query2.Result.State, ShouldEqual, models.AlertNotificationStatePending) + + Convey("Update existing state to completed should update database", func() { + s := *cmd.State + cmd := models.SetAlertNotificationStateToCompleteCommand{ + State: &s, + } + err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + + query3 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetAlertNotificationState(context.Background(), query3) + So(err, ShouldBeNil) + So(query3.Result.Version, ShouldEqual, 2) + So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + }) + + Convey("Update existing state to completed should update database, but return version mismatch", func() { + cmd.State.Version = 1000 + s := *cmd.State + cmd := models.SetAlertNotificationStateToCompleteCommand{ + State: &s, + } + err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) + So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + + query3 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetAlertNotificationState(context.Background(), query3) + So(err, ShouldBeNil) + So(query3.Result.Version, ShouldEqual, 1001) + So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + }) + }) + + Convey("Update existing state to pending with incorrect version should return version mismatch error", func() { + s := *query.Result + s.Version = 1000 + cmd := models.SetAlertNotificationStateToPendingCommand{ + State: &s, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + }) }) }) - //Convey("Can insert new state for alert notifier", func() { - // createCmd := &models.InsertAlertNotificationCommand{ - // AlertId: alertId, - // NotifierId: notifierId, - // OrgId: orgId, - // SentAt: 1, - // State: models.AlertNotificationStateCompleted, - // } - // - // err := InsertAlertNotificationState(context.Background(), createCmd) - // So(err, ShouldBeNil) - // - // err = InsertAlertNotificationState(context.Background(), createCmd) - // So(err, ShouldEqual, models.ErrAlertNotificationStateAlreadyExist) - // - // Convey("should be able to update alert notifier state", func() { - // updateCmd := &models.SetAlertNotificationStateToPendingCommand{ - // State: models.AlertNotificationState{ - // Id: 1, - // SentAt: 1, - // Version: 0, - // } - // } - // - // err := SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) - // So(err, ShouldBeNil) - // - // Convey("should not be able to set pending on old version", func() { - // err = SetAlertNotificationStateToPendingCommand(context.Background(), updateCmd) - // So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) - // }) - // - // Convey("should be able to set state to completed", func() { - // cmd := &models.SetAlertNotificationStateToCompleteCommand{Id: 1} - // err = SetAlertNotificationStateToCompleteCommand(context.Background(), cmd) - // So(err, ShouldBeNil) - // }) - // }) - // }) - //}) - Convey("Alert notifications should be empty", func() { cmd := &models.GetAlertNotificationsQuery{ OrgId: 2, From 67c58fa5e40e78da554afbe257ba73da426f1dab Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 15:11:03 +0200 Subject: [PATCH 11/28] fix set sent_at on complete --- pkg/services/alerting/notifier.go | 2 +- pkg/services/sqlstore/alert_notification.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index a80fb265e81..4f69514977a 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -68,7 +68,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no if err != nil { n.log.Error("failed to send notification", "id", not.GetNotifierId()) } else { - notifierState.state.SentAt = time.Now().Unix() + notifierState.state.SentAt = time.Now().UTC().Unix() } if evalContext.IsTestRun { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index fdf467695dc..f93ef7b8164 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -264,11 +264,12 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA sql := `UPDATE alert_notification_state SET state = ?, - version = ? + version = ?, + sent_at = ? WHERE id = ?` - _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.Id) + _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.SentAt, cmd.State.Id) if err != nil { return err From 8551ffa0b003270da3121df2eb1110d9710d172f Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 18:34:20 +0200 Subject: [PATCH 12/28] alert -> ok with reminders enabled should send --- pkg/services/alerting/notifiers/base.go | 25 ++++--- pkg/services/alerting/notifiers/base_test.go | 79 +++++++++++--------- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index e1fc2969154..6dce8494569 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -51,19 +51,26 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ return false } - // Do not notify if interval has not elapsed - lastNotify := time.Unix(notificationState.SentAt, 0) - if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { - return false - } + if context.PrevAlertState == context.Rule.State && sendReminder { + // Do not notify if interval has not elapsed + lastNotify := time.Unix(notificationState.SentAt, 0) + if !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { + return false + } - // Do not notify if alert state if OK or pending even on repeated notify - if sendReminder && (context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending) { - return false + // Do not notify if alert state is OK or pending even on repeated notify + if context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending { + return false + } } // Do not notify when we become OK for the first time. - if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) { + if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK { + return false + } + + // Do not notify when we OK -> Pending + if context.PrevAlertState == models.AlertStateOK && context.Rule.State == models.AlertStatePending { return false } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 50cfbef7387..3645255e385 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -20,34 +20,34 @@ func TestShouldSendAlertNotification(t *testing.T) { newState m.AlertStateType sendReminder bool frequency time.Duration - journals *m.AlertNotificationState + state *m.AlertNotificationState expect bool }{ { name: "pending -> ok should not trigger an notification", - newState: m.AlertStatePending, - prevState: m.AlertStateOK, + newState: m.AlertStateOK, + prevState: m.AlertStatePending, sendReminder: false, - journals: &m.AlertNotificationState{}, + state: &m.AlertNotificationState{}, expect: false, }, { name: "ok -> alerting should trigger an notification", - newState: m.AlertStateOK, - prevState: m.AlertStateAlerting, + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, sendReminder: false, - journals: &m.AlertNotificationState{}, + state: &m.AlertNotificationState{}, expect: true, }, { name: "ok -> pending should not trigger an notification", - newState: m.AlertStateOK, - prevState: m.AlertStatePending, + newState: m.AlertStatePending, + prevState: m.AlertStateOK, sendReminder: false, - journals: &m.AlertNotificationState{}, + state: &m.AlertNotificationState{}, expect: false, }, @@ -56,66 +56,77 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: false, - journals: &m.AlertNotificationState{}, + state: &m.AlertNotificationState{}, expect: false, }, - { - name: "ok -> alerting should trigger an notification", - newState: m.AlertStateOK, - prevState: m.AlertStateAlerting, - sendReminder: true, - journals: &m.AlertNotificationState{}, - - expect: true, - }, { name: "ok -> ok with reminder should not trigger an notification", newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: true, - journals: &m.AlertNotificationState{}, + state: &m.AlertNotificationState{}, expect: false, }, { - name: "alerting -> alerting with reminder and no journaling should trigger", - newState: m.AlertStateAlerting, + name: "alerting -> ok should trigger an notification", + newState: m.AlertStateOK, prevState: m.AlertStateAlerting, - frequency: time.Minute * 10, - sendReminder: true, - journals: &m.AlertNotificationState{}, + sendReminder: false, + state: &m.AlertNotificationState{}, expect: true, }, { - name: "alerting -> alerting with reminder and successful recent journal event should not trigger", + name: "alerting -> ok should trigger an notification when reminders enabled", + newState: m.AlertStateOK, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, + + expect: true, + }, + { + name: "alerting -> alerting with reminder and no state should trigger", newState: m.AlertStateAlerting, prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - journals: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, + state: &m.AlertNotificationState{}, + + expect: true, + }, + { + name: "alerting -> alerting with reminder and last notification sent 1 minute ago should not trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, expect: false, }, { - name: "alerting -> alerting with reminder and failed recent journal event should trigger", + name: "alerting -> alerting with reminder and last notifciation sent 11 minutes ago should trigger", newState: m.AlertStateAlerting, prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - expect: true, - journals: &m.AlertNotificationState{SentAt: tnow.Add(-time.Hour).Unix()}, + state: &m.AlertNotificationState{SentAt: tnow.Add(-11 * time.Minute).Unix()}, + + expect: true, }, } for _, tc := range tcs { evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ - State: tc.newState, + State: tc.prevState, }) - evalContext.Rule.State = tc.prevState - if defaultShouldNotify(evalContext, true, tc.frequency, tc.journals) != tc.expect { + evalContext.Rule.State = tc.newState + if defaultShouldNotify(evalContext, tc.sendReminder, tc.frequency, tc.state) != tc.expect { t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect) } } From d313ffa8470ed9c2ea8366d97e9adbf260bfb626 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 28 Sep 2018 18:35:43 +0200 Subject: [PATCH 13/28] devenv: enable some debug logging for ha test setup --- devenv/docker/ha_test/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 78f98ab8dc5..1ca984d68c8 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -34,6 +34,7 @@ services: - GF_DATABASE_PASSWORD=password - GF_SESSION_PROVIDER=mysql - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(mysql:3306)/grafana?allowNativePasswords=true + - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug ports: - 3000 depends_on: From d412aafb7e364807b7b6a3856a31d094d33ee573 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Sun, 30 Sep 2018 20:16:01 +0200 Subject: [PATCH 14/28] remove unused code --- pkg/models/alert_notifications.go | 7 ------- pkg/services/sqlstore/alert_notification.go | 23 --------------------- 2 files changed, 30 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 14bf8694207..bf52e10499a 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -111,10 +111,3 @@ type GetNotificationStateQuery struct { Result *AlertNotificationState } -type InsertAlertNotificationCommand struct { - OrgId int64 - AlertId int64 - NotifierId int64 - SentAt int64 - State AlertNotificationStateType -} diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index f93ef7b8164..97db569c214 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -19,7 +19,6 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) - bus.AddHandlerCtx("sql", InsertAlertNotificationState) bus.AddHandlerCtx("sql", GetAlertNotificationState) bus.AddHandlerCtx("sql", SetAlertNotificationStateToCompleteCommand) bus.AddHandlerCtx("sql", SetAlertNotificationStateToPendingCommand) @@ -231,28 +230,6 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { }) } -func InsertAlertNotificationState(ctx context.Context, cmd *m.InsertAlertNotificationCommand) error { - return withDbSession(ctx, func(sess *DBSession) error { - notificationState := &m.AlertNotificationState{ - OrgId: cmd.OrgId, - AlertId: cmd.AlertId, - NotifierId: cmd.NotifierId, - SentAt: cmd.SentAt, - State: cmd.State, - } - - if _, err := sess.Insert(notificationState); err != nil { - if dialect.IsUniqueConstraintViolation(err) { - return m.ErrAlertNotificationStateAlreadyExist - } - - return err - } - - return nil - }) -} - func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { return withDbSession(ctx, func(sess *DBSession) error { version := cmd.State.Version From 5ec086dc56807acd1929ba110e92812650b0ecc1 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Sun, 30 Sep 2018 21:52:50 +0200 Subject: [PATCH 15/28] don't notify if notification state pending If notification state is pending and last update of state was made less than a minute ago. In the case of a grafana instance is shut down/crashes between setting pending state and before sending the notification/marks as complete this logic should allow the notification to be sent after some time instead of being left in an inconsistent state where no notifications are being sent. --- pkg/models/alert_notifications.go | 2 +- pkg/services/alerting/notifiers/base.go | 10 +++++++++- pkg/services/alerting/notifiers/base_test.go | 16 ++++++++++++++++ pkg/services/sqlstore/alert_notification.go | 13 ++++++++----- pkg/services/sqlstore/alert_notification_test.go | 12 ++++++++++++ pkg/services/sqlstore/migrations/alert_mig.go | 1 + 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index bf52e10499a..8608303ddde 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -93,6 +93,7 @@ type AlertNotificationState struct { SentAt int64 State AlertNotificationStateType Version int64 + UpdatedAt int64 } type SetAlertNotificationStateToPendingCommand struct { @@ -110,4 +111,3 @@ type GetNotificationStateQuery struct { Result *AlertNotificationState } - diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 6dce8494569..b13725f1138 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -54,7 +54,7 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ if context.PrevAlertState == context.Rule.State && sendReminder { // Do not notify if interval has not elapsed lastNotify := time.Unix(notificationState.SentAt, 0) - if !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { + if notificationState.SentAt != 0 && lastNotify.Add(frequency).After(time.Now()) { return false } @@ -74,6 +74,14 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ return false } + // Do not notifu if state pending and it have been updated last minute + if notificationState.State == models.AlertNotificationStatePending { + lastUpdated := time.Unix(notificationState.UpdatedAt, 0) + if lastUpdated.Add(1 * time.Minute).After(time.Now()) { + return false + } + } + return true } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 3645255e385..581ff6550db 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -116,6 +116,22 @@ func TestShouldSendAlertNotification(t *testing.T) { sendReminder: true, state: &m.AlertNotificationState{SentAt: tnow.Add(-11 * time.Minute).Unix()}, + expect: true, + }, + { + name: "OK -> alerting with notifciation state pending and updated 30 seconds ago should not trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, + state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-30 * time.Second).Unix()}, + + expect: false, + }, + { + name: "OK -> alerting with notifciation state pending and updated 2 minutes ago should trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, + state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()}, + expect: true, }, } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 97db569c214..a69168d094a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -242,11 +242,12 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA sql := `UPDATE alert_notification_state SET state = ?, version = ?, - sent_at = ? + sent_at = ?, + updated_at = ? WHERE id = ?` - _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.SentAt, cmd.State.Id) + _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.SentAt, timeNow().Unix(), cmd.State.Id) if err != nil { return err @@ -268,12 +269,13 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl sql := `UPDATE alert_notification_state SET state = ?, - version = ? + version = ?, + updated_at = ? WHERE id = ? AND version = ?` - res, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.Id, currentVersion) + res, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, timeNow().Unix(), cmd.State.Id, currentVersion) if err != nil { return err @@ -310,6 +312,7 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ AlertId: cmd.AlertId, NotifierId: cmd.NotifierId, State: "unknown", + UpdatedAt: timeNow().Unix(), } if _, err := sess.Insert(notificationState); err != nil { @@ -337,7 +340,7 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ } func getAlertNotificationState(sess *DBSession, cmd *m.GetNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { - exist, err := sess.Desc("alert_notification_state.sent_at"). + exist, err := sess. Where("alert_notification_state.org_id = ?", cmd.OrgId). Where("alert_notification_state.alert_id = ?", cmd.AlertId). Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index daed5a8cd7f..f82022fd18b 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -18,6 +18,9 @@ func TestAlertNotificationSQLAccess(t *testing.T) { var alertID int64 = 7 var orgID int64 = 5 var notifierID int64 = 10 + oldTimeNow := timeNow + now := time.Date(2018, 9, 30, 0, 0, 0, 0, time.UTC) + timeNow = func() time.Time { return now } Convey("Get no existing state should create a new state", func() { query := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} @@ -26,6 +29,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(query.Result, ShouldNotBeNil) So(query.Result.State, ShouldEqual, "unknown") So(query.Result.Version, ShouldEqual, 0) + So(query.Result.UpdatedAt, ShouldEqual, now.Unix()) Convey("Get existing state should not create a new state", func() { query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} @@ -33,6 +37,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) So(query2.Result, ShouldNotBeNil) So(query2.Result.Id, ShouldEqual, query.Result.Id) + So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) }) Convey("Update existing state to pending with correct version should update database", func() { @@ -50,6 +55,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) So(query2.Result.Version, ShouldEqual, 1) So(query2.Result.State, ShouldEqual, models.AlertNotificationStatePending) + So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) Convey("Update existing state to completed should update database", func() { s := *cmd.State @@ -64,6 +70,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) So(query3.Result.Version, ShouldEqual, 2) So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) }) Convey("Update existing state to completed should update database, but return version mismatch", func() { @@ -80,6 +87,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldBeNil) So(query3.Result.Version, ShouldEqual, 1001) So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) }) }) @@ -93,6 +101,10 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) }) }) + + Reset(func() { + timeNow = oldTimeNow + }) }) Convey("Alert notifications should be empty", func() { diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 877dafcf1e1..bd42bb0343d 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -120,6 +120,7 @@ func addAlertMigrations(mg *Migrator) { {Name: "sent_at", Type: DB_BigInt, Nullable: false}, {Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false}, {Name: "version", Type: DB_BigInt, Nullable: false}, + {Name: "updated_at", Type: DB_BigInt, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex}, From 1be8fb76b8d7f71525a64b66c7b34836deca5515 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Sun, 30 Sep 2018 21:57:15 +0200 Subject: [PATCH 16/28] cleanup alert_notification_state when deleting alert rules and channels --- pkg/services/sqlstore/alert.go | 4 ++++ pkg/services/sqlstore/alert_notification.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index ba898769578..b8206db191a 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -60,6 +60,10 @@ func deleteAlertByIdInternal(alertId int64, reason string, sess *DBSession) erro return err } + if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_id = ?", alertId); err != nil { + return err + } + return nil } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index a69168d094a..1fc0394414b 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -27,8 +27,15 @@ func init() { func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { return inTransaction(func(sess *DBSession) error { sql := "DELETE FROM alert_notification WHERE alert_notification.org_id = ? AND alert_notification.id = ?" - _, err := sess.Exec(sql, cmd.OrgId, cmd.Id) - return err + if _, err := sess.Exec(sql, cmd.OrgId, cmd.Id); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_notification_state.org_id = ? AND alert_notification_state.notifier_id = ?", cmd.OrgId, cmd.Id); err != nil { + return err + } + + return nil }) } From 75f832cda8ef15e72c700a903ecc6cbb81349d79 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 1 Oct 2018 14:13:03 +0200 Subject: [PATCH 17/28] use alert state changes counter as secondary version --- pkg/models/alert.go | 4 ++-- pkg/models/alert_notifications.go | 3 ++- pkg/services/alerting/notifier.go | 5 ++-- pkg/services/alerting/result_handler.go | 5 ++++ pkg/services/alerting/rule.go | 2 ++ pkg/services/sqlstore/alert.go | 2 ++ pkg/services/sqlstore/alert_notification.go | 15 ++++++++---- .../sqlstore/alert_notification_test.go | 23 +++++++++++++++++++ pkg/services/sqlstore/migrations/alert_mig.go | 4 ++++ 9 files changed, 54 insertions(+), 9 deletions(-) diff --git a/pkg/models/alert.go b/pkg/models/alert.go index fba2aa63df9..ba1fc0779ba 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -75,7 +75,7 @@ type Alert struct { EvalData *simplejson.Json NewStateDate time.Time - StateChanges int + StateChanges int64 Created time.Time Updated time.Time @@ -156,7 +156,7 @@ type SetAlertStateCommand struct { Error string EvalData *simplejson.Json - Timestamp time.Time + Result Alert } //Queries diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 8608303ddde..2d50185e33d 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -97,7 +97,8 @@ type AlertNotificationState struct { } type SetAlertNotificationStateToPendingCommand struct { - State *AlertNotificationState + AlertRuleStateUpdatedVersion int64 + State *AlertNotificationState } type SetAlertNotificationStateToCompleteCommand struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 4f69514977a..cc60e61fb4c 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -94,7 +94,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { if !evalContext.IsTestRun { setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ - State: notifierState.state, + State: notifierState.state, + AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges, } err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd) @@ -172,7 +173,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] for _, notification := range query.Result { not, err := n.createNotifierFor(notification) if err != nil { - n.log.Error("Could not create notifier", "notifier", notification.Id) + n.log.Error("Could not create notifier", "notifier", notification.Id, "error", err) continue } diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index e2c70de0e28..455296fbfc7 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -67,6 +67,11 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } handler.log.Error("Failed to save state", "error", err) + } else { + + // StateChanges is used for de dupping alert notifications + // when two servers are raising. + evalContext.Rule.StateChanges = cmd.Result.StateChanges } // save annotation diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 018d138dbe4..ce135ea31eb 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,6 +23,8 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 + + StateChanges int64 } type ValidationError struct { diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index b8206db191a..2f17402b80c 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -279,6 +279,8 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error { } sess.ID(alert.Id).Update(&alert) + + cmd.Result = alert return nil }) } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 1fc0394414b..01a81023dff 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -277,19 +277,26 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl sql := `UPDATE alert_notification_state SET state = ?, version = ?, - updated_at = ? + updated_at = ?, + alert_rule_state_updated_version = ? WHERE id = ? AND - version = ?` + (version = ? OR alert_rule_state_updated_version < ?)` - res, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, timeNow().Unix(), cmd.State.Id, currentVersion) + res, err := sess.Exec(sql, + cmd.State.State, + cmd.State.Version, + timeNow().Unix(), + cmd.AlertRuleStateUpdatedVersion, + cmd.State.Id, + currentVersion, + cmd.AlertRuleStateUpdatedVersion) if err != nil { return err } affected, _ := res.RowsAffected() - if affected == 0 { return m.ErrAlertNotificationStateVersionConflict } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index f82022fd18b..9bcf2c18f4d 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -100,6 +100,29 @@ func TestAlertNotificationSQLAccess(t *testing.T) { err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) }) + + Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() { + s := *query.Result + cmd := models.SetAlertNotificationStateToPendingCommand{ + State: &s, + AlertRuleStateUpdatedVersion: 1000, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + + So(cmd.State.Version, ShouldEqual, 1) + So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending) + }) + + Convey("different version and same alert state change version should return error", func() { + s := *query.Result + s.Version = 1000 + cmd := models.SetAlertNotificationStateToPendingCommand{ + State: &s, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldNotBeNil) + }) }) Reset(func() { diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index bd42bb0343d..5b76f0273fd 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -130,4 +130,8 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0])) + + mg.AddMigration("Add alert_rule_state_updated_version to alert_notification_state", NewAddColumnMigration(alert_notification_state, &Column{ + Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: true, + })) } From 35eddee2dd4bb701009866bbcb199c13c17c7048 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 1 Oct 2018 19:40:03 +0200 Subject: [PATCH 18/28] devenv: add postgres ha test config example --- devenv/docker/ha_test/docker-compose.yaml | 32 ++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 1ca984d68c8..ce8630d88a4 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -8,18 +8,33 @@ services: volumes: - /var/run/docker.sock:/tmp/docker.sock:ro - mysql: + db: image: mysql environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: grafana MYSQL_USER: grafana MYSQL_PASSWORD: password + ports: + - 3306 healthcheck: test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] timeout: 10s retries: 10 + # db: + # image: postgres:9.3 + # environment: + # POSTGRES_DATABASE: grafana + # POSTGRES_USER: grafana + # POSTGRES_PASSWORD: password + # ports: + # - 5432 + # healthcheck: + # test: ["CMD-SHELL", "pg_isready -d grafana -U grafana"] + # timeout: 10s + # retries: 10 + grafana: image: grafana/grafana:dev volumes: @@ -27,18 +42,23 @@ services: environment: - VIRTUAL_HOST=grafana.loc - GF_SERVER_ROOT_URL=http://grafana.loc - - GF_DATABASE_TYPE=mysql - - GF_DATABASE_HOST=mysql:3306 - GF_DATABASE_NAME=grafana - GF_DATABASE_USER=grafana - GF_DATABASE_PASSWORD=password + - GF_DATABASE_TYPE=mysql + - GF_DATABASE_HOST=db:3306 - GF_SESSION_PROVIDER=mysql - - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(mysql:3306)/grafana?allowNativePasswords=true + - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true + # - GF_DATABASE_TYPE=postgres + # - GF_DATABASE_HOST=db:5432 + # - GF_DATABASE_SSL_MODE=disable + # - GF_SESSION_PROVIDER=postgres + # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug ports: - 3000 depends_on: - mysql: + db: condition: service_healthy prometheus: @@ -55,4 +75,4 @@ services: # environment: # - DATA_SOURCE_NAME=grafana:password@(mysql:3306)/ # ports: - # - 9104 \ No newline at end of file + # - 9104 From 7f1d7cefc0ec2eed37e3f58c4ff8a2da1a44e713 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 11:19:09 +0200 Subject: [PATCH 19/28] reminder: uses UpdatedAt to track state changes. --- pkg/models/alert_notifications.go | 17 +++++++++-------- pkg/services/alerting/notifier.go | 4 ++-- pkg/services/alerting/notifiers/base.go | 4 ++-- pkg/services/alerting/notifiers/base_test.go | 6 +++--- pkg/services/alerting/rule.go | 1 + pkg/services/sqlstore/alert_notification.go | 8 +++----- pkg/services/sqlstore/migrations/alert_mig.go | 6 +----- 7 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 2d50185e33d..c44f3a35e0c 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -19,6 +19,7 @@ type AlertNotificationStateType string var ( AlertNotificationStatePending = AlertNotificationStateType("pending") AlertNotificationStateCompleted = AlertNotificationStateType("completed") + AlertNotificationStateUnknown = AlertNotificationStateType("unknown") ) type AlertNotification struct { @@ -86,14 +87,14 @@ type GetAllAlertNotificationsQuery struct { } type AlertNotificationState struct { - Id int64 - OrgId int64 - AlertId int64 - NotifierId int64 - SentAt int64 - State AlertNotificationStateType - Version int64 - UpdatedAt int64 + Id int64 + OrgId int64 + AlertId int64 + NotifierId int64 + State AlertNotificationStateType + Version int64 + UpdatedAt int64 + AlertRuleStateUpdatedVersion int64 } type SetAlertNotificationStateToPendingCommand struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index cc60e61fb4c..cd31c9f64af 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -68,7 +68,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no if err != nil { n.log.Error("failed to send notification", "id", not.GetNotifierId()) } else { - notifierState.state.SentAt = time.Now().UTC().Unix() + notifierState.state.UpdatedAt = time.Now().UTC().Unix() } if evalContext.IsTestRun { @@ -185,7 +185,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] err = bus.DispatchCtx(evalContext.Ctx, query) if err != nil { - n.log.Error("Could not get notification state.", "notifier", notification.Id) + n.log.Error("Could not get notification state.", "notifier", notification.Id, "error", err) continue } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index b13725f1138..9fe940f70dd 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -53,8 +53,8 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ if context.PrevAlertState == context.Rule.State && sendReminder { // Do not notify if interval has not elapsed - lastNotify := time.Unix(notificationState.SentAt, 0) - if notificationState.SentAt != 0 && lastNotify.Add(frequency).After(time.Now()) { + lastNotify := time.Unix(notificationState.UpdatedAt, 0) + if notificationState.UpdatedAt != 0 && lastNotify.Add(frequency).After(time.Now()) { return false } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 581ff6550db..778be41a93f 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -84,7 +84,7 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - state: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, expect: true, }, @@ -104,7 +104,7 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - state: &m.AlertNotificationState{SentAt: tnow.Add(-time.Minute).Unix()}, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, expect: false, }, @@ -114,7 +114,7 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - state: &m.AlertNotificationState{SentAt: tnow.Add(-11 * time.Minute).Unix()}, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-11 * time.Minute).Unix()}, expect: true, }, diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index ce135ea31eb..d13924c2a17 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -102,6 +102,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) + model.StateChanges = ruleDef.StateChanges for _, v := range ruleDef.Settings.Get("notifications").MustArray() { jsonModel := simplejson.NewFromAny(v) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 01a81023dff..e55c783c63a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -249,12 +249,11 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA sql := `UPDATE alert_notification_state SET state = ?, version = ?, - sent_at = ?, updated_at = ? WHERE id = ?` - _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, cmd.State.SentAt, timeNow().Unix(), cmd.State.Id) + _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, timeNow().Unix(), cmd.State.Id) if err != nil { return err @@ -325,7 +324,7 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ OrgId: cmd.OrgId, AlertId: cmd.AlertId, NotifierId: cmd.NotifierId, - State: "unknown", + State: m.AlertNotificationStateUnknown, UpdatedAt: timeNow().Unix(), } @@ -354,10 +353,9 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ } func getAlertNotificationState(sess *DBSession, cmd *m.GetNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { - exist, err := sess. + return sess. Where("alert_notification_state.org_id = ?", cmd.OrgId). Where("alert_notification_state.alert_id = ?", cmd.AlertId). Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). Get(nj) - return exist, err } diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 5b76f0273fd..cadcccf6c95 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -117,10 +117,10 @@ func addAlertMigrations(mg *Migrator) { {Name: "org_id", Type: DB_BigInt, Nullable: false}, {Name: "alert_id", Type: DB_BigInt, Nullable: false}, {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, - {Name: "sent_at", Type: DB_BigInt, Nullable: false}, {Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false}, {Name: "version", Type: DB_BigInt, Nullable: false}, {Name: "updated_at", Type: DB_BigInt, Nullable: false}, + {Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex}, @@ -130,8 +130,4 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0])) - - mg.AddMigration("Add alert_rule_state_updated_version to alert_notification_state", NewAddColumnMigration(alert_notification_state, &Column{ - Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: true, - })) } From 341d8af637d6801ca277d50239e6c54d10e93ee6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 11:23:40 +0200 Subject: [PATCH 20/28] rename GetNotificationStateQuery to GetOrCreateNotificationStateQuery --- pkg/models/alert_notifications.go | 2 +- pkg/services/alerting/notifier.go | 2 +- pkg/services/sqlstore/alert_notification.go | 6 +++--- .../sqlstore/alert_notification_test.go | 20 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index c44f3a35e0c..d908200b36e 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -106,7 +106,7 @@ type SetAlertNotificationStateToCompleteCommand struct { State *AlertNotificationState } -type GetNotificationStateQuery struct { +type GetOrCreateNotificationStateQuery struct { OrgId int64 AlertId int64 NotifierId int64 diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index cd31c9f64af..4f866a58ab7 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -177,7 +177,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] continue } - query := &m.GetNotificationStateQuery{ + query := &m.GetOrCreateNotificationStateQuery{ NotifierId: notification.Id, AlertId: evalContext.Rule.Id, OrgId: evalContext.Rule.OrgId, diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index e55c783c63a..33d19570a3c 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -19,7 +19,7 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) - bus.AddHandlerCtx("sql", GetAlertNotificationState) + bus.AddHandlerCtx("sql", GetOrCreateAlertNotificationState) bus.AddHandlerCtx("sql", SetAlertNotificationStateToCompleteCommand) bus.AddHandlerCtx("sql", SetAlertNotificationStateToPendingCommand) } @@ -304,7 +304,7 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl }) } -func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQuery) error { +func GetOrCreateAlertNotificationState(ctx context.Context, cmd *m.GetOrCreateNotificationStateQuery) error { return withDbSession(ctx, func(sess *DBSession) error { nj := &m.AlertNotificationState{} @@ -352,7 +352,7 @@ func GetAlertNotificationState(ctx context.Context, cmd *m.GetNotificationStateQ }) } -func getAlertNotificationState(sess *DBSession, cmd *m.GetNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { +func getAlertNotificationState(sess *DBSession, cmd *m.GetOrCreateNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { return sess. Where("alert_notification_state.org_id = ?", cmd.OrgId). Where("alert_notification_state.alert_id = ?", cmd.AlertId). diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 9bcf2c18f4d..559a1379f91 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -23,8 +23,8 @@ func TestAlertNotificationSQLAccess(t *testing.T) { timeNow = func() time.Time { return now } Convey("Get no existing state should create a new state", func() { - query := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} - err := GetAlertNotificationState(context.Background(), query) + query := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetOrCreateAlertNotificationState(context.Background(), query) So(err, ShouldBeNil) So(query.Result, ShouldNotBeNil) So(query.Result.State, ShouldEqual, "unknown") @@ -32,8 +32,8 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(query.Result.UpdatedAt, ShouldEqual, now.Unix()) Convey("Get existing state should not create a new state", func() { - query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} - err := GetAlertNotificationState(context.Background(), query2) + query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetOrCreateAlertNotificationState(context.Background(), query2) So(err, ShouldBeNil) So(query2.Result, ShouldNotBeNil) So(query2.Result.Id, ShouldEqual, query.Result.Id) @@ -50,8 +50,8 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(cmd.State.Version, ShouldEqual, 1) So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending) - query2 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} - err = GetAlertNotificationState(context.Background(), query2) + query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query2) So(err, ShouldBeNil) So(query2.Result.Version, ShouldEqual, 1) So(query2.Result.State, ShouldEqual, models.AlertNotificationStatePending) @@ -65,8 +65,8 @@ func TestAlertNotificationSQLAccess(t *testing.T) { err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) So(err, ShouldBeNil) - query3 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} - err = GetAlertNotificationState(context.Background(), query3) + query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query3) So(err, ShouldBeNil) So(query3.Result.Version, ShouldEqual, 2) So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) @@ -82,8 +82,8 @@ func TestAlertNotificationSQLAccess(t *testing.T) { err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) - query3 := &models.GetNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} - err = GetAlertNotificationState(context.Background(), query3) + query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query3) So(err, ShouldBeNil) So(query3.Result.Version, ShouldEqual, 1001) So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) From 9022e871e3cb3c5384415a2eab768f678dda471a Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 13:57:41 +0200 Subject: [PATCH 21/28] avoid sending full notification state to pending/complete --- pkg/models/alert_notifications.go | 8 +++- pkg/services/alerting/notifier.go | 10 ++++- pkg/services/sqlstore/alert_notification.go | 24 ++++++------ .../sqlstore/alert_notification_test.go | 38 +++++++++++-------- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index d908200b36e..2128b469fa4 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -98,12 +98,16 @@ type AlertNotificationState struct { } type SetAlertNotificationStateToPendingCommand struct { + Id int64 AlertRuleStateUpdatedVersion int64 - State *AlertNotificationState + Version int64 + + ResultVersion int64 } type SetAlertNotificationStateToCompleteCommand struct { - State *AlertNotificationState + Id int64 + Version int64 } type GetOrCreateNotificationStateQuery struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 4f866a58ab7..dae310a8e3f 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -76,7 +76,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no } cmd := &m.SetAlertNotificationStateToCompleteCommand{ - State: notifierState.state, + Id: not.GetNotifierId(), + Version: notifierState.state.Version, } if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { @@ -94,7 +95,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { if !evalContext.IsTestRun { setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ - State: notifierState.state, + Id: notifierState.state.NotifierId, + Version: notifierState.state.Version, AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges, } @@ -106,6 +108,10 @@ func (n *notificationService) sendNotification(evalContext *EvalContext, notifie if err != nil { return err } + + // We need to update state version to be able to log + // unexpected version conflicts when marking notifications as ok + notifierState.state.Version = setPendingCmd.ResultVersion } return n.sendAndMarkAsComplete(evalContext, notifierState) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 33d19570a3c..069e6213b03 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -239,12 +239,11 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - version := cmd.State.Version + version := cmd.Version var current m.AlertNotificationState - sess.ID(cmd.State.Id).Get(¤t) + sess.ID(cmd.Id).Get(¤t) - cmd.State.State = m.AlertNotificationStateCompleted - cmd.State.Version++ + newVersion := cmd.Version + 1 sql := `UPDATE alert_notification_state SET state = ?, @@ -253,7 +252,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA WHERE id = ?` - _, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, timeNow().Unix(), cmd.State.Id) + _, err := sess.Exec(sql, m.AlertNotificationStateCompleted, newVersion, timeNow().Unix(), cmd.Id) if err != nil { return err @@ -269,10 +268,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { return withDbSession(ctx, func(sess *DBSession) error { - currentVersion := cmd.State.Version - cmd.State.State = m.AlertNotificationStatePending - cmd.State.Version++ - + newVersion := cmd.Version + 1 sql := `UPDATE alert_notification_state SET state = ?, version = ?, @@ -283,12 +279,12 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl (version = ? OR alert_rule_state_updated_version < ?)` res, err := sess.Exec(sql, - cmd.State.State, - cmd.State.Version, + m.AlertNotificationStatePending, + newVersion, timeNow().Unix(), cmd.AlertRuleStateUpdatedVersion, - cmd.State.Id, - currentVersion, + cmd.Id, + cmd.Version, cmd.AlertRuleStateUpdatedVersion) if err != nil { @@ -300,6 +296,8 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl return m.ErrAlertNotificationStateVersionConflict } + cmd.ResultVersion = newVersion + return nil }) } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 559a1379f91..df360d14efa 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -42,13 +42,16 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Update existing state to pending with correct version should update database", func() { s := *query.Result + cmd := models.SetAlertNotificationStateToPendingCommand{ - State: &s, + Id: s.Id, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) So(err, ShouldBeNil) - So(cmd.State.Version, ShouldEqual, 1) - So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending) + So(cmd.ResultVersion, ShouldEqual, 1) query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} err = GetOrCreateAlertNotificationState(context.Background(), query2) @@ -58,11 +61,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) Convey("Update existing state to completed should update database", func() { - s := *cmd.State - cmd := models.SetAlertNotificationStateToCompleteCommand{ - State: &s, + s := *query.Result + setStateCmd := models.SetAlertNotificationStateToCompleteCommand{ + Id: s.Id, + Version: cmd.ResultVersion, } - err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) + err := SetAlertNotificationStateToCompleteCommand(context.Background(), &setStateCmd) So(err, ShouldBeNil) query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} @@ -74,10 +78,10 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Update existing state to completed should update database, but return version mismatch", func() { - cmd.State.Version = 1000 - s := *cmd.State + s := *query.Result cmd := models.SetAlertNotificationStateToCompleteCommand{ - State: &s, + Id: s.Id, + Version: 1000, } err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) @@ -95,7 +99,9 @@ func TestAlertNotificationSQLAccess(t *testing.T) { s := *query.Result s.Version = 1000 cmd := models.SetAlertNotificationStateToPendingCommand{ - State: &s, + Id: s.NotifierId, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, } err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) @@ -104,21 +110,23 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() { s := *query.Result cmd := models.SetAlertNotificationStateToPendingCommand{ - State: &s, + Id: s.Id, + Version: s.Version, AlertRuleStateUpdatedVersion: 1000, } err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) So(err, ShouldBeNil) - So(cmd.State.Version, ShouldEqual, 1) - So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending) + So(cmd.ResultVersion, ShouldEqual, 1) }) Convey("different version and same alert state change version should return error", func() { s := *query.Result s.Version = 1000 cmd := models.SetAlertNotificationStateToPendingCommand{ - State: &s, + Id: s.Id, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, } err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) So(err, ShouldNotBeNil) From b45a1bd45cec04555b04bd9989470a163ac6080c Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 14:03:30 +0200 Subject: [PATCH 22/28] avoid exporting notificationState and notificationStateSlice --- pkg/services/alerting/interfaces.go | 6 +++--- pkg/services/alerting/notifier.go | 12 ++++++------ pkg/services/alerting/test_notification.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 65369ed3884..96294f0624f 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -30,14 +30,14 @@ type Notifier interface { GetFrequency() time.Duration } -type NotifierState struct { +type notifierState struct { notifier Notifier state *models.AlertNotificationState } -type NotifierStateSlice []*NotifierState +type notifierStateSlice []*notifierState -func (notifiers NotifierStateSlice) ShouldUploadImage() bool { +func (notifiers notifierStateSlice) ShouldUploadImage() bool { for _, ns := range notifiers { if ns.notifier.NeedsImage() { return true diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index dae310a8e3f..cf488b08e4f 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -58,7 +58,7 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { return n.sendNotifications(context, notifierStates) } -func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *NotifierState) error { +func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *notifierState) error { not := notifierState.notifier n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() @@ -92,7 +92,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no return nil } -func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { +func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *notifierState) error { if !evalContext.IsTestRun { setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ Id: notifierState.state.NotifierId, @@ -117,7 +117,7 @@ func (n *notificationService) sendNotification(evalContext *EvalContext, notifie return n.sendAndMarkAsComplete(evalContext, notifierState) } -func (n *notificationService) sendNotifications(evalContext *EvalContext, notifierStates NotifierStateSlice) error { +func (n *notificationService) sendNotifications(evalContext *EvalContext, notifierStates notifierStateSlice) error { for _, notifierState := range notifierStates { err := n.sendNotification(evalContext, notifierState) if err != nil { @@ -168,14 +168,14 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { return nil } -func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierStateSlice, error) { +func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (notifierStateSlice, error) { query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds} if err := bus.Dispatch(query); err != nil { return nil, err } - var result NotifierStateSlice + var result notifierStateSlice for _, notification := range query.Result { not, err := n.createNotifierFor(notification) if err != nil { @@ -196,7 +196,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] } if not.ShouldNotify(evalContext.Ctx, evalContext, query.Result) { - result = append(result, &NotifierState{ + result = append(result, ¬ifierState{ notifier: not, state: query.Result, }) diff --git a/pkg/services/alerting/test_notification.go b/pkg/services/alerting/test_notification.go index 228ec90001d..8aa1b80aa22 100644 --- a/pkg/services/alerting/test_notification.go +++ b/pkg/services/alerting/test_notification.go @@ -39,7 +39,7 @@ func handleNotificationTestCommand(cmd *NotificationTestCommand) error { return err } - return notifier.sendNotifications(createTestEvalContext(cmd), NotifierStateSlice{{notifier: notifiers}}) + return notifier.sendNotifications(createTestEvalContext(cmd), notifierStateSlice{{notifier: notifiers}}) } func createTestEvalContext(cmd *NotificationTestCommand) *EvalContext { From 9d3659d30d5cc10fd415700e3ad7df50408f6f82 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 14:04:50 +0200 Subject: [PATCH 23/28] improve local variable name --- pkg/services/alerting/notifier.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index cf488b08e4f..6a9b2d0b3ef 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -59,14 +59,14 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { } func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *notifierState) error { - not := notifierState.notifier - n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + notifier := notifierState.notifier + n.log.Debug("Sending notification", "type", notifier.GetType(), "id", notifier.GetNotifierId(), "isDefault", notifier.GetIsDefault()) + metrics.M_Alerting_Notification_Sent.WithLabelValues(notifier.GetType()).Inc() - err := not.Notify(evalContext) + err := notifier.Notify(evalContext) if err != nil { - n.log.Error("failed to send notification", "id", not.GetNotifierId()) + n.log.Error("failed to send notification", "id", notifier.GetNotifierId()) } else { notifierState.state.UpdatedAt = time.Now().UTC().Unix() } @@ -76,13 +76,13 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no } cmd := &m.SetAlertNotificationStateToCompleteCommand{ - Id: not.GetNotifierId(), + Id: notifier.GetNotifierId(), Version: notifierState.state.Version, } if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { if err == m.ErrAlertNotificationStateVersionConflict { - n.log.Error("notification state out of sync", "id", not.GetNotifierId()) + n.log.Error("notification state out of sync", "id", notifier.GetNotifierId()) return nil } From 67e5f625149b02c245a277798ad3129494b9ec89 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 14:23:18 +0200 Subject: [PATCH 24/28] move version conflict logging for mark as complete to sqlstore package --- pkg/services/alerting/notifier.go | 17 +++-------------- pkg/services/sqlstore/alert_notification.go | 5 ++++- .../sqlstore/alert_notification_test.go | 9 +++++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 6a9b2d0b3ef..0f3290c0dbe 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -3,7 +3,6 @@ package alerting import ( "errors" "fmt" - "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" @@ -60,15 +59,14 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *notifierState) error { notifier := notifierState.notifier + n.log.Debug("Sending notification", "type", notifier.GetType(), "id", notifier.GetNotifierId(), "isDefault", notifier.GetIsDefault()) metrics.M_Alerting_Notification_Sent.WithLabelValues(notifier.GetType()).Inc() err := notifier.Notify(evalContext) if err != nil { - n.log.Error("failed to send notification", "id", notifier.GetNotifierId()) - } else { - notifierState.state.UpdatedAt = time.Now().UTC().Unix() + n.log.Error("failed to send notification", "id", notifier.GetNotifierId(), "error", err) } if evalContext.IsTestRun { @@ -80,16 +78,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no Version: notifierState.state.Version, } - if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { - if err == m.ErrAlertNotificationStateVersionConflict { - n.log.Error("notification state out of sync", "id", notifier.GetNotifierId()) - return nil - } - - return err - } - - return nil + return bus.DispatchCtx(evalContext.Ctx, cmd) } func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *notifierState) error { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 069e6213b03..8815f4c4bd7 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -259,7 +259,10 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA } if current.Version != version { - return m.ErrAlertNotificationStateVersionConflict + sqlog.Error( + `notification state out of sync. the notification + is marked as complete but has been modified between + set as pending and completion.`, "notifierId", current.NotifierId) } return nil diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index df360d14efa..ed682bae5c6 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -77,19 +77,20 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) }) - Convey("Update existing state to completed should update database, but return version mismatch", func() { + Convey("Update existing state to completed should update database. regardless of version", func() { s := *query.Result + unknownVersion := int64(1000) cmd := models.SetAlertNotificationStateToCompleteCommand{ Id: s.Id, - Version: 1000, + Version: unknownVersion, } err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) - So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + So(err, ShouldBeNil) query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} err = GetOrCreateAlertNotificationState(context.Background(), query3) So(err, ShouldBeNil) - So(query3.Result.Version, ShouldEqual, 1001) + So(query3.Result.Version, ShouldEqual, unknownVersion+1) So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) }) From 9289cba6252ffa3fc1a01bc2ef4150475ba84fdd Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 14:28:48 +0200 Subject: [PATCH 25/28] merges defaultShouldNotify and ShouldNotify --- pkg/services/alerting/notifiers/base.go | 20 ++++++++------------ pkg/services/alerting/notifiers/base_test.go | 4 +++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 9fe940f70dd..fbade2eccac 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -45,16 +45,17 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase { } } -func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, notificationState *models.AlertNotificationState) bool { +// ShouldNotify checks this evaluation should send an alert notification +func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State && !sendReminder { + if context.PrevAlertState == context.Rule.State && !n.SendReminder { return false } - if context.PrevAlertState == context.Rule.State && sendReminder { + if context.PrevAlertState == context.Rule.State && n.SendReminder { // Do not notify if interval has not elapsed - lastNotify := time.Unix(notificationState.UpdatedAt, 0) - if notificationState.UpdatedAt != 0 && lastNotify.Add(frequency).After(time.Now()) { + lastNotify := time.Unix(notiferState.UpdatedAt, 0) + if notiferState.UpdatedAt != 0 && lastNotify.Add(n.Frequency).After(time.Now()) { return false } @@ -75,8 +76,8 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ } // Do not notifu if state pending and it have been updated last minute - if notificationState.State == models.AlertNotificationStatePending { - lastUpdated := time.Unix(notificationState.UpdatedAt, 0) + if notiferState.State == models.AlertNotificationStatePending { + lastUpdated := time.Unix(notiferState.UpdatedAt, 0) if lastUpdated.Add(1 * time.Minute).After(time.Now()) { return false } @@ -85,11 +86,6 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ return true } -// ShouldNotify checks this evaluation should send an alert notification -func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { - return defaultShouldNotify(c, n.SendReminder, n.Frequency, notiferState) -} - func (n *NotifierBase) GetType() string { return n.Type } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 778be41a93f..5e46d3ad72e 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -142,7 +142,9 @@ func TestShouldSendAlertNotification(t *testing.T) { }) evalContext.Rule.State = tc.newState - if defaultShouldNotify(evalContext, tc.sendReminder, tc.frequency, tc.state) != tc.expect { + nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency} + + if nb.ShouldNotify(evalContext.Ctx, evalContext, tc.state) != tc.expect { t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect) } } From 7b781e4c3c6490211e7b731ea0567cb18d0011a7 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 2 Oct 2018 14:53:39 +0200 Subject: [PATCH 26/28] use notification state id instead of notifier id --- pkg/services/alerting/notifier.go | 6 +++--- pkg/services/sqlstore/alert_notification.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 0f3290c0dbe..9ce50eadd6b 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -74,7 +74,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no } cmd := &m.SetAlertNotificationStateToCompleteCommand{ - Id: notifier.GetNotifierId(), + Id: notifierState.state.Id, Version: notifierState.state.Version, } @@ -84,7 +84,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *notifierState) error { if !evalContext.IsTestRun { setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ - Id: notifierState.state.NotifierId, + Id: notifierState.state.Id, Version: notifierState.state.Version, AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges, } @@ -110,7 +110,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi for _, notifierState := range notifierStates { err := n.sendNotification(evalContext, notifierState) if err != nil { - n.log.Error("failed to send notification", "id", notifierState.notifier.GetNotifierId()) + n.log.Error("failed to send notification", "id", notifierState.notifier.GetNotifierId(), "error", err) } } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 8815f4c4bd7..665b4b0cd0a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -238,7 +238,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { } func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { - return withDbSession(ctx, func(sess *DBSession) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { version := cmd.Version var current m.AlertNotificationState sess.ID(cmd.Id).Get(¤t) From 418f77e70ef48b9fae6975c6e0c43ce8dc12333d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 2 Oct 2018 15:07:44 +0200 Subject: [PATCH 27/28] get or create alert notification should use transaction --- pkg/services/sqlstore/alert_notification.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 665b4b0cd0a..daaef945b96 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -259,10 +259,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA } if current.Version != version { - sqlog.Error( - `notification state out of sync. the notification - is marked as complete but has been modified between - set as pending and completion.`, "notifierId", current.NotifierId) + sqlog.Error("notification state out of sync. the notification is marked as complete but has been modified between set as pending and completion.", "notifierId", current.NotifierId) } return nil @@ -306,7 +303,7 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl } func GetOrCreateAlertNotificationState(ctx context.Context, cmd *m.GetOrCreateNotificationStateQuery) error { - return withDbSession(ctx, func(sess *DBSession) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { nj := &m.AlertNotificationState{} exist, err := getAlertNotificationState(sess, cmd, nj) From 0e0901874b56fd27a001c4070c72cd63dd1c5173 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 2 Oct 2018 15:11:33 +0200 Subject: [PATCH 28/28] better comment about state changes --- pkg/services/alerting/result_handler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 455296fbfc7..420ffeb9a55 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -69,8 +69,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { handler.log.Error("Failed to save state", "error", err) } else { - // StateChanges is used for de dupping alert notifications - // when two servers are raising. + // StateChanges is used for de duping alert notifications + // when two servers are raising. This makes sure that the server + // with the last state change always sends a notification. evalContext.Rule.StateChanges = cmd.Result.StateChanges }