Merge pull request #13440 from grafana/reminder_refactoring

Transaction issues for alert reminder
This commit is contained in:
Marcus Efraimsson
2018-10-02 15:55:18 +02:00
committed by GitHub
23 changed files with 1668 additions and 277 deletions
+6
View File
@@ -60,6 +60,10 @@ func deleteAlertByIdInternal(alertId int64, reason string, sess *DBSession) erro
return err
}
if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_id = ?", alertId); err != nil {
return err
}
return nil
}
@@ -275,6 +279,8 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error {
}
sess.ID(alert.Id).Update(&alert)
cmd.Result = alert
return nil
})
}
+119 -32
View File
@@ -3,6 +3,7 @@ package sqlstore
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"time"
@@ -18,16 +19,23 @@ func init() {
bus.AddHandler("sql", DeleteAlertNotification)
bus.AddHandler("sql", GetAlertNotificationsToSend)
bus.AddHandler("sql", GetAllAlertNotifications)
bus.AddHandlerCtx("sql", RecordNotificationJournal)
bus.AddHandlerCtx("sql", GetLatestNotification)
bus.AddHandlerCtx("sql", CleanNotificationJournal)
bus.AddHandlerCtx("sql", GetOrCreateAlertNotificationState)
bus.AddHandlerCtx("sql", SetAlertNotificationStateToCompleteCommand)
bus.AddHandlerCtx("sql", SetAlertNotificationStateToPendingCommand)
}
func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
return inTransaction(func(sess *DBSession) error {
sql := "DELETE FROM alert_notification WHERE alert_notification.org_id = ? AND alert_notification.id = ?"
_, err := sess.Exec(sql, cmd.OrgId, cmd.Id)
return err
if _, err := sess.Exec(sql, cmd.OrgId, cmd.Id); err != nil {
return err
}
if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_notification_state.org_id = ? AND alert_notification_state.notifier_id = ?", cmd.OrgId, cmd.Id); err != nil {
return err
}
return nil
})
}
@@ -229,44 +237,123 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
})
}
func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJournalCommand) error {
return withDbSession(ctx, func(sess *DBSession) error {
journalEntry := &m.AlertNotificationJournal{
OrgId: cmd.OrgId,
AlertId: cmd.AlertId,
NotifierId: cmd.NotifierId,
SentAt: cmd.SentAt,
Success: cmd.Success,
}
func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error {
return inTransactionCtx(ctx, func(sess *DBSession) error {
version := cmd.Version
var current m.AlertNotificationState
sess.ID(cmd.Id).Get(&current)
_, err := sess.Insert(journalEntry)
return err
})
}
newVersion := cmd.Version + 1
func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
return withDbSession(ctx, func(sess *DBSession) error {
nj := []m.AlertNotificationJournal{}
sql := `UPDATE alert_notification_state SET
state = ?,
version = ?,
updated_at = ?
WHERE
id = ?`
err := sess.Desc("alert_notification_journal.sent_at").
Where("alert_notification_journal.org_id = ?", cmd.OrgId).
Where("alert_notification_journal.alert_id = ?", cmd.AlertId).
Where("alert_notification_journal.notifier_id = ?", cmd.NotifierId).
Find(&nj)
_, err := sess.Exec(sql, m.AlertNotificationStateCompleted, newVersion, timeNow().Unix(), cmd.Id)
if err != nil {
return err
}
cmd.Result = nj
if current.Version != version {
sqlog.Error("notification state out of sync. the notification is marked as complete but has been modified between set as pending and completion.", "notifierId", current.NotifierId)
}
return nil
})
}
func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error {
return inTransactionCtx(ctx, func(sess *DBSession) error {
sql := "DELETE FROM alert_notification_journal WHERE alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?"
_, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId)
return err
func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error {
return withDbSession(ctx, func(sess *DBSession) error {
newVersion := cmd.Version + 1
sql := `UPDATE alert_notification_state SET
state = ?,
version = ?,
updated_at = ?,
alert_rule_state_updated_version = ?
WHERE
id = ? AND
(version = ? OR alert_rule_state_updated_version < ?)`
res, err := sess.Exec(sql,
m.AlertNotificationStatePending,
newVersion,
timeNow().Unix(),
cmd.AlertRuleStateUpdatedVersion,
cmd.Id,
cmd.Version,
cmd.AlertRuleStateUpdatedVersion)
if err != nil {
return err
}
affected, _ := res.RowsAffected()
if affected == 0 {
return m.ErrAlertNotificationStateVersionConflict
}
cmd.ResultVersion = newVersion
return nil
})
}
func GetOrCreateAlertNotificationState(ctx context.Context, cmd *m.GetOrCreateNotificationStateQuery) error {
return inTransactionCtx(ctx, func(sess *DBSession) error {
nj := &m.AlertNotificationState{}
exist, err := getAlertNotificationState(sess, cmd, nj)
// if exists, return it, otherwise create it with default values
if err != nil {
return err
}
if exist {
cmd.Result = nj
return nil
}
notificationState := &m.AlertNotificationState{
OrgId: cmd.OrgId,
AlertId: cmd.AlertId,
NotifierId: cmd.NotifierId,
State: m.AlertNotificationStateUnknown,
UpdatedAt: timeNow().Unix(),
}
if _, err := sess.Insert(notificationState); err != nil {
if dialect.IsUniqueConstraintViolation(err) {
exist, err = getAlertNotificationState(sess, cmd, nj)
if err != nil {
return err
}
if !exist {
return errors.New("Should not happen")
}
cmd.Result = nj
return nil
}
return err
}
cmd.Result = notificationState
return nil
})
}
func getAlertNotificationState(sess *DBSession, cmd *m.GetOrCreateNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) {
return sess.
Where("alert_notification_state.org_id = ?", cmd.OrgId).
Where("alert_notification_state.alert_id = ?", cmd.AlertId).
Where("alert_notification_state.notifier_id = ?", cmd.NotifierId).
Get(nj)
}
+126 -51
View File
@@ -6,7 +6,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
@@ -14,58 +14,133 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
Convey("Testing Alert notification sql access", t, func() {
InitTestDB(t)
Convey("Alert notification journal", func() {
var alertId int64 = 7
var orgId int64 = 5
var notifierId int64 = 10
Convey("Alert notification state", func() {
var alertID int64 = 7
var orgID int64 = 5
var notifierID int64 = 10
oldTimeNow := timeNow
now := time.Date(2018, 9, 30, 0, 0, 0, 0, time.UTC)
timeNow = func() time.Time { return now }
Convey("Getting last journal should raise error if no one exists", func() {
query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId}
GetLatestNotification(context.Background(), query)
So(len(query.Result), ShouldEqual, 0)
// recording an journal entry in another org to make sure org filter works as expected.
journalInOtherOrg := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: 10, Success: true, SentAt: 1}
err := RecordNotificationJournal(context.Background(), journalInOtherOrg)
Convey("Get no existing state should create a new state", func() {
query := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err := GetOrCreateAlertNotificationState(context.Background(), query)
So(err, ShouldBeNil)
So(query.Result, ShouldNotBeNil)
So(query.Result.State, ShouldEqual, "unknown")
So(query.Result.Version, ShouldEqual, 0)
So(query.Result.UpdatedAt, ShouldEqual, now.Unix())
Convey("should be able to record two journaling events", func() {
createCmd := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, Success: true, SentAt: 1}
err := RecordNotificationJournal(context.Background(), createCmd)
Convey("Get existing state should not create a new state", func() {
query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err := GetOrCreateAlertNotificationState(context.Background(), query2)
So(err, ShouldBeNil)
So(query2.Result, ShouldNotBeNil)
So(query2.Result.Id, ShouldEqual, query.Result.Id)
So(query2.Result.UpdatedAt, ShouldEqual, now.Unix())
})
createCmd.SentAt += 1000 //increase epoch
Convey("Update existing state to pending with correct version should update database", func() {
s := *query.Result
err = RecordNotificationJournal(context.Background(), createCmd)
cmd := models.SetAlertNotificationStateToPendingCommand{
Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
}
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldBeNil)
So(cmd.ResultVersion, ShouldEqual, 1)
Convey("get last journaling event", func() {
err := GetLatestNotification(context.Background(), query)
query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err = GetOrCreateAlertNotificationState(context.Background(), query2)
So(err, ShouldBeNil)
So(query2.Result.Version, ShouldEqual, 1)
So(query2.Result.State, ShouldEqual, models.AlertNotificationStatePending)
So(query2.Result.UpdatedAt, ShouldEqual, now.Unix())
Convey("Update existing state to completed should update database", func() {
s := *query.Result
setStateCmd := models.SetAlertNotificationStateToCompleteCommand{
Id: s.Id,
Version: cmd.ResultVersion,
}
err := SetAlertNotificationStateToCompleteCommand(context.Background(), &setStateCmd)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 2)
last := query.Result[0]
So(last.SentAt, ShouldEqual, 1001)
Convey("be able to clear all journaling for an notifier", func() {
cmd := &m.CleanNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId}
err := CleanNotificationJournal(context.Background(), cmd)
So(err, ShouldBeNil)
query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err = GetOrCreateAlertNotificationState(context.Background(), query3)
So(err, ShouldBeNil)
So(query3.Result.Version, ShouldEqual, 2)
So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted)
So(query3.Result.UpdatedAt, ShouldEqual, now.Unix())
})
Convey("querying for last journaling should return no journal entries", func() {
query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId}
err := GetLatestNotification(context.Background(), query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
})
Convey("Update existing state to completed should update database. regardless of version", func() {
s := *query.Result
unknownVersion := int64(1000)
cmd := models.SetAlertNotificationStateToCompleteCommand{
Id: s.Id,
Version: unknownVersion,
}
err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd)
So(err, ShouldBeNil)
query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err = GetOrCreateAlertNotificationState(context.Background(), query3)
So(err, ShouldBeNil)
So(query3.Result.Version, ShouldEqual, unknownVersion+1)
So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted)
So(query3.Result.UpdatedAt, ShouldEqual, now.Unix())
})
})
Convey("Update existing state to pending with incorrect version should return version mismatch error", func() {
s := *query.Result
s.Version = 1000
cmd := models.SetAlertNotificationStateToPendingCommand{
Id: s.NotifierId,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
}
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict)
})
Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() {
s := *query.Result
cmd := models.SetAlertNotificationStateToPendingCommand{
Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: 1000,
}
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldBeNil)
So(cmd.ResultVersion, ShouldEqual, 1)
})
Convey("different version and same alert state change version should return error", func() {
s := *query.Result
s.Version = 1000
cmd := models.SetAlertNotificationStateToPendingCommand{
Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
}
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldNotBeNil)
})
})
Reset(func() {
timeNow = oldTimeNow
})
})
Convey("Alert notifications should be empty", func() {
cmd := &m.GetAlertNotificationsQuery{
cmd := &models.GetAlertNotificationsQuery{
OrgId: 2,
Name: "email",
}
@@ -76,7 +151,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Cannot save alert notifier with send reminder = true", func() {
cmd := &m.CreateAlertNotificationCommand{
cmd := &models.CreateAlertNotificationCommand{
Name: "ops",
Type: "email",
OrgId: 1,
@@ -86,7 +161,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
Convey("and missing frequency", func() {
err := CreateAlertNotificationCommand(cmd)
So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound)
So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound)
})
Convey("invalid frequency", func() {
@@ -98,7 +173,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Cannot update alert notifier with send reminder = false", func() {
cmd := &m.CreateAlertNotificationCommand{
cmd := &models.CreateAlertNotificationCommand{
Name: "ops update",
Type: "email",
OrgId: 1,
@@ -109,14 +184,14 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
err := CreateAlertNotificationCommand(cmd)
So(err, ShouldBeNil)
updateCmd := &m.UpdateAlertNotificationCommand{
updateCmd := &models.UpdateAlertNotificationCommand{
Id: cmd.Result.Id,
SendReminder: true,
}
Convey("and missing frequency", func() {
err := UpdateAlertNotification(updateCmd)
So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound)
So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound)
})
Convey("invalid frequency", func() {
@@ -129,7 +204,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Can save Alert Notification", func() {
cmd := &m.CreateAlertNotificationCommand{
cmd := &models.CreateAlertNotificationCommand{
Name: "ops",
Type: "email",
OrgId: 1,
@@ -151,7 +226,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Can update alert notification", func() {
newCmd := &m.UpdateAlertNotificationCommand{
newCmd := &models.UpdateAlertNotificationCommand{
Name: "NewName",
Type: "webhook",
OrgId: cmd.Result.OrgId,
@@ -167,7 +242,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Can update alert notification to disable sending of reminders", func() {
newCmd := &m.UpdateAlertNotificationCommand{
newCmd := &models.UpdateAlertNotificationCommand{
Name: "NewName",
Type: "webhook",
OrgId: cmd.Result.OrgId,
@@ -182,12 +257,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("Can search using an array of ids", func() {
cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd1 := models.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd2 := models.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd3 := models.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
cmd4 := models.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
otherOrg := models.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil)
So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil)
@@ -196,7 +271,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
So(CreateAlertNotificationCommand(&otherOrg), ShouldBeNil)
Convey("search", func() {
query := &m.GetAlertNotificationsToSendQuery{
query := &models.GetAlertNotificationsToSendQuery{
Ids: []int64{cmd1.Result.Id, cmd2.Result.Id, 112341231},
OrgId: 1,
}
@@ -207,7 +282,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
})
Convey("all", func() {
query := &m.GetAllAlertNotificationsQuery{
query := &models.GetAllAlertNotificationsQuery{
OrgId: 1,
}
@@ -107,4 +107,27 @@ func addAlertMigrations(mg *Migrator) {
mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))
mg.AddMigration("drop alert_notification_journal", NewDropTableMigration("alert_notification_journal"))
alert_notification_state := Table{
Name: "alert_notification_state",
Columns: []*Column{
{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "org_id", Type: DB_BigInt, Nullable: false},
{Name: "alert_id", Type: DB_BigInt, Nullable: false},
{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
{Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false},
{Name: "version", Type: DB_BigInt, Nullable: false},
{Name: "updated_at", Type: DB_BigInt, Nullable: false},
{Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: false},
},
Indices: []*Index{
{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex},
},
}
mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
}
@@ -44,6 +44,8 @@ type Dialect interface {
CleanDB() error
NoOpSql() string
IsUniqueConstraintViolation(err error) bool
}
func NewDialect(engine *xorm.Engine) Dialect {
@@ -5,6 +5,8 @@ import (
"strconv"
"strings"
"github.com/VividCortex/mysqlerr"
"github.com/go-sql-driver/mysql"
"github.com/go-xorm/xorm"
)
@@ -125,3 +127,13 @@ func (db *Mysql) CleanDB() error {
return nil
}
func (db *Mysql) IsUniqueConstraintViolation(err error) bool {
if driverErr, ok := err.(*mysql.MySQLError); ok {
if driverErr.Number == mysqlerr.ER_DUP_ENTRY {
return true
}
}
return false
}
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/go-xorm/xorm"
"github.com/lib/pq"
)
type Postgres struct {
@@ -136,3 +137,13 @@ func (db *Postgres) CleanDB() error {
return nil
}
func (db *Postgres) IsUniqueConstraintViolation(err error) bool {
if driverErr, ok := err.(*pq.Error); ok {
if driverErr.Code == "23505" {
return true
}
}
return false
}
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/go-xorm/xorm"
sqlite3 "github.com/mattn/go-sqlite3"
)
type Sqlite3 struct {
@@ -82,3 +83,13 @@ func (db *Sqlite3) DropIndexSql(tableName string, index *Index) string {
func (db *Sqlite3) CleanDB() error {
return nil
}
func (db *Sqlite3) IsUniqueConstraintViolation(err error) bool {
if driverErr, ok := err.(sqlite3.Error); ok {
if driverErr.ExtendedCode == sqlite3.ErrConstraintUnique {
return true
}
}
return false
}