From 8b91e57ef6490208eaa78fce66bf53f69048734c Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 13 Jun 2016 16:39:00 +0200 Subject: [PATCH 01/41] feat(alerting): notification query --- pkg/models/alert_notifications.go | 44 +++++++++++++++ pkg/services/alerting/alert_rule.go | 2 + pkg/services/sqlstore/alert_notification.go | 56 +++++++++++++++++++ .../sqlstore/alert_notification_test.go | 36 ++++++++++++ pkg/services/sqlstore/migrations/alert_mig.go | 15 +++++ 5 files changed, 153 insertions(+) create mode 100644 pkg/models/alert_notifications.go create mode 100644 pkg/services/sqlstore/alert_notification.go create mode 100644 pkg/services/sqlstore/alert_notification_test.go diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go new file mode 100644 index 00000000000..b0dcfcfb82e --- /dev/null +++ b/pkg/models/alert_notifications.go @@ -0,0 +1,44 @@ +package models + +import ( + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" +) + +type AlertNotification struct { + Id int64 + OrgId int64 + Name string + Type string + Settings *simplejson.Json + + Created time.Time + Updated time.Time +} + +type CreateAlertNotificationCommand struct { + Name string + Type string + OrgID int64 + Settings *simplejson.Json + + Result *AlertNotification +} + +type UpdateAlertNotificationCommand struct { + Name string + Type string + OrgID int64 + Settings *simplejson.Json + + Result *AlertNotification +} + +type GetAlertNotificationQuery struct { + Name string + ID int64 + OrgID int64 + + Result []*AlertNotification +} diff --git a/pkg/services/alerting/alert_rule.go b/pkg/services/alerting/alert_rule.go index 6e460a65dfd..25284b755a7 100644 --- a/pkg/services/alerting/alert_rule.go +++ b/pkg/services/alerting/alert_rule.go @@ -26,6 +26,8 @@ type AlertRule struct { Transform string TransformParams simplejson.Json Transformer transformers.Transformer + + NotificationGroups []int64 } var ( diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go new file mode 100644 index 00000000000..5a173c548b7 --- /dev/null +++ b/pkg/services/sqlstore/alert_notification.go @@ -0,0 +1,56 @@ +package sqlstore + +import ( + "bytes" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" +) + +func init() { + bus.AddHandler("sql", GetAlertNotifications) +} + +func GetAlertNotifications(query *m.GetAlertNotificationQuery) error { + var sql bytes.Buffer + params := make([]interface{}, 0) + + sql.WriteString(`SELECT + alert_notification.id, + alert_notification.org_id, + alert_notification.name, + alert_notification.type, + alert_notification.created, + alert_notification.updated, + alert_notification.settings + FROM alert_notification + `) + + sql.WriteString(` WHERE alert_notification.org_id = ?`) + params = append(params, query.OrgID) + + if query.Name != "" { + sql.WriteString(` AND alert_notification.name = ?`) + params = append(params, query.Name) + } + + var result []*m.AlertNotification + if err := x.Sql(sql.String(), params...).Find(&result); err != nil { + return err + } + + query.Result = result + return nil +} + +/* +func CreateAlertNotification(cmd *m.CreateAlertNotificationCommand) error { + return inTransaction(func(sess *xorm.Session) error { + + + }) +} + +func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { + +}*/ diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go new file mode 100644 index 00000000000..332394e97be --- /dev/null +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -0,0 +1,36 @@ +package sqlstore + +import ( + "fmt" + "testing" + + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestAlertNotificationSQLAccess(t *testing.T) { + Convey("Testing Alert notification sql access", t, func() { + InitTestDB(t) + + Convey("Alert notifications should be empty", func() { + cmd := &m.GetAlertNotificationQuery{ + OrgID: FakeOrgId, + Name: "email", + } + + err := GetAlertNotifications(cmd) + fmt.Printf("errror %v", err) + So(err, ShouldBeNil) + So(len(cmd.Result), ShouldEqual, 0) + }) + /* + Convey("Can save Alert Notification", func() { + cmd := &m.CreateAlertNotificationCommand{} + + var err error + err = CreateAlertNotification(cmd) + + So(err, ShouldBeNil) + }) */ + }) +} diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index ff9ad5abf51..2ff7712f8bb 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -65,4 +65,19 @@ func addAlertMigrations(mg *Migrator) { } mg.AddMigration("create alert_heartbeat table v1", NewAddTableMigration(alert_heartbeat)) + + alert_notification := Table{ + Name: "alert_notification", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "settings", Type: DB_Text, Nullable: false}, + {Name: "created", Type: DB_DateTime, Nullable: false}, + {Name: "updated", Type: DB_DateTime, Nullable: false}, + }, + } + + mg.AddMigration("create alert_notification table v1", NewAddTableMigration(alert_notification)) } From dbf3795aaffe46cd3f624ef5270fff6e5342f5bf Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 14 Jun 2016 08:33:50 +0200 Subject: [PATCH 02/41] feat(alerting): add sql layer for alert notifications --- pkg/models/alert_notifications.go | 1 + pkg/services/sqlstore/alert_notification.go | 73 +++++++++++++++++-- .../sqlstore/alert_notification_test.go | 45 ++++++++++-- 3 files changed, 106 insertions(+), 13 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index b0dcfcfb82e..5f6fb6b5018 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -27,6 +27,7 @@ type CreateAlertNotificationCommand struct { } type UpdateAlertNotificationCommand struct { + Id int64 Name string Type string OrgID int64 diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 5a173c548b7..d64edf3a043 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -2,16 +2,25 @@ package sqlstore import ( "bytes" + "fmt" + "time" + "github.com/go-xorm/xorm" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" ) func init() { - bus.AddHandler("sql", GetAlertNotifications) + bus.AddHandler("sql", AlertNotificationQuery) + bus.AddHandler("sql", CreateAlertNotificationCommand) + bus.AddHandler("sql", UpdateAlertNotification) } -func GetAlertNotifications(query *m.GetAlertNotificationQuery) error { +func AlertNotificationQuery(query *m.GetAlertNotificationQuery) error { + return getAlertNotifications(query, x.NewSession()) +} + +func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Session) error { var sql bytes.Buffer params := make([]interface{}, 0) @@ -35,7 +44,7 @@ func GetAlertNotifications(query *m.GetAlertNotificationQuery) error { } var result []*m.AlertNotification - if err := x.Sql(sql.String(), params...).Find(&result); err != nil { + if err := sess.Sql(sql.String(), params...).Find(&result); err != nil { return err } @@ -43,14 +52,66 @@ func GetAlertNotifications(query *m.GetAlertNotificationQuery) error { return nil } -/* -func CreateAlertNotification(cmd *m.CreateAlertNotificationCommand) error { +func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error { return inTransaction(func(sess *xorm.Session) error { + existingQuery := &m.GetAlertNotificationQuery{OrgID: cmd.OrgID, Name: cmd.Name} + err := getAlertNotifications(existingQuery, sess) + if err != nil { + return err + } + if len(existingQuery.Result) > 0 { + return fmt.Errorf("Alert notification name %s already exists", cmd.Name) + } + + alertNotification := &m.AlertNotification{ + OrgId: cmd.OrgID, + Name: cmd.Name, + Type: cmd.Type, + Created: time.Now(), + Settings: cmd.Settings, + } + + id, err := sess.Insert(alertNotification) + + if err != nil { + return err + } + + alertNotification.Id = id + cmd.Result = alertNotification + return nil }) } func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { + return inTransaction(func(sess *xorm.Session) (err error) { + an := &m.AlertNotification{} -}*/ + var has bool + has, err = sess.Id(cmd.Id).Get(an) + + if err != nil { + return err + } + + if !has { + return fmt.Errorf("Alert notification does not exist") + } + + an.Name = cmd.Name + an.Type = cmd.Type + an.Settings = cmd.Settings + an.Updated = time.Now() + + _, err = sess.Id(an.Id).Cols("name", "type", "settings", "updated").Update(an) + + if err != nil { + return err + } + + cmd.Result = an + return nil + }) +} diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 332394e97be..48dcd23b751 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -11,6 +11,7 @@ import ( func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) + var err error Convey("Alert notifications should be empty", func() { cmd := &m.GetAlertNotificationQuery{ @@ -18,19 +19,49 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Name: "email", } - err := GetAlertNotifications(cmd) + err := AlertNotificationQuery(cmd) fmt.Printf("errror %v", err) So(err, ShouldBeNil) So(len(cmd.Result), ShouldEqual, 0) }) - /* - Convey("Can save Alert Notification", func() { - cmd := &m.CreateAlertNotificationCommand{} - var err error - err = CreateAlertNotification(cmd) + Convey("Can save Alert Notification", func() { + cmd := &m.CreateAlertNotificationCommand{ + Name: "ops", + Type: "email", + } + err = CreateAlertNotificationCommand(cmd) + So(err, ShouldBeNil) + So(cmd.Result.Id, ShouldNotEqual, 0) + + Convey("Cannot save Alert Notification with the same name", func() { + err = CreateAlertNotificationCommand(cmd) + So(err, ShouldNotBeNil) + }) + + Convey("Cannot update alert notification that does not exist", func() { + newCmd := &m.UpdateAlertNotificationCommand{ + Name: "NewName", + Type: cmd.Result.Type, + OrgID: cmd.Result.OrgId, + Id: 1337, + } + err = UpdateAlertNotification(newCmd) + So(err, ShouldNotBeNil) + }) + + Convey("Can update alert notification", func() { + newCmd := &m.UpdateAlertNotificationCommand{ + Name: "NewName", + Type: cmd.Result.Type, + OrgID: cmd.Result.OrgId, + Id: cmd.Result.Id, + } + err = UpdateAlertNotification(newCmd) So(err, ShouldBeNil) - }) */ + So(newCmd.Result.Name, ShouldEqual, "NewName") + }) + }) }) } From 6eca26e8ec5954486e07de39eba7a16fef5b82e7 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 14 Jun 2016 08:47:42 +0200 Subject: [PATCH 03/41] style(alerting): improve formating --- pkg/services/sqlstore/alert_notification.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index d64edf3a043..de2222c995f 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -87,10 +87,10 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return inTransaction(func(sess *xorm.Session) (err error) { - an := &m.AlertNotification{} + alertNotification := &m.AlertNotification{} var has bool - has, err = sess.Id(cmd.Id).Get(an) + has, err = sess.Id(cmd.Id).Get(alertNotification) if err != nil { return err @@ -100,18 +100,18 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return fmt.Errorf("Alert notification does not exist") } - an.Name = cmd.Name - an.Type = cmd.Type - an.Settings = cmd.Settings - an.Updated = time.Now() + alertNotification.Name = cmd.Name + alertNotification.Type = cmd.Type + alertNotification.Settings = cmd.Settings + alertNotification.Updated = time.Now() - _, err = sess.Id(an.Id).Cols("name", "type", "settings", "updated").Update(an) + _, err = sess.Id(alertNotification.Id).Cols("name", "type", "settings", "updated").Update(alertNotification) if err != nil { return err } - cmd.Result = an + cmd.Result = alertNotification return nil }) } From 9a8416416dc967cb51382d3d74984f5e261e20c9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 14 Jun 2016 16:56:14 +0200 Subject: [PATCH 04/41] feat(alerting): converter for db model to notification --- pkg/models/alert_notifications.go | 3 +- pkg/services/alerting/notifier.go | 113 ++++++++++++++++++ pkg/services/alerting/notifier_test.go | 68 +++++++++++ pkg/services/sqlstore/alert_notification.go | 47 ++++++-- .../sqlstore/alert_notification_test.go | 58 ++++++--- 5 files changed, 259 insertions(+), 30 deletions(-) create mode 100644 pkg/services/alerting/notifier_test.go diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 5f6fb6b5018..7c22351b172 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -38,7 +38,8 @@ type UpdateAlertNotificationCommand struct { type GetAlertNotificationQuery struct { Name string - ID int64 + Id int64 + Ids []int64 OrgID int64 Result []*AlertNotification diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index d806a5d69ca..a843a19b0eb 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1 +1,114 @@ package alerting + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting/alertstates" +) + +type Notifier struct { + log log.Logger +} + +func NewNotifier() *Notifier { + return &Notifier{ + log: log.New("alerting.notifier"), + } +} + +func (n *Notifier) Notify(alertResult AlertResult) { + notifiers := getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) + + for _, notifier := range notifiers { + warn := alertResult.State == alertstates.Warn && notifier.SendWarning + crit := alertResult.State == alertstates.Critical && notifier.SendCritical + + if warn || crit { + n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) + go notifier.Notifierr.Notify(alertResult) + } + } + +} + +type Notification struct { + Name string + Type string + SendWarning bool + SendCritical bool + + Notifierr Notifierr +} + +type EmailNotifier struct { + To string + From string +} + +func (this EmailNotifier) Notify(alertResult AlertResult) { + //bus.dispath to notification package in grafana +} + +type WebhookNotifier struct { + Url string + AuthUser string + AuthPassword string +} + +func (this WebhookNotifier) Notify(alertResult AlertResult) { + //bus.dispath to notification package in grafana +} + +type Notifierr interface { + Notify(alertResult AlertResult) +} + +func getNotifiers(orgId int64, notificationGroups []int64) []*Notification { + var notifications []*m.AlertNotification + + for _, notificationId := range notificationGroups { + query := m.GetAlertNotificationQuery{ + OrgID: orgId, + Id: notificationId, + } + + notifications = append(notifications, query.Result...) + } + + var result []*Notification + + for _, notification := range notifications { + not, err := NewNotificationFromDBModel(notification) + if err == nil { + result = append(result, not) + } + } + + return result +} + +func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, error) { + return &Notification{ + Name: model.Name, + Type: model.Type, + Notifierr: createNotifier(model.Type, model.Settings), + SendCritical: !model.Settings.Get("ignoreCrit").MustBool(), + SendWarning: !model.Settings.Get("ignoreWarn").MustBool(), + }, nil +} + +var createNotifier = func(notificationType string, settings *simplejson.Json) Notifierr { + if notificationType == "email" { + return &EmailNotifier{ + To: settings.Get("to").MustString(), + From: settings.Get("from").MustString(), + } + } + + return &WebhookNotifier{ + Url: settings.Get("url").MustString(), + AuthUser: settings.Get("user").MustString(), + AuthPassword: settings.Get("password").MustString(), + } +} diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go new file mode 100644 index 00000000000..23af5eb41cf --- /dev/null +++ b/pkg/services/alerting/notifier_test.go @@ -0,0 +1,68 @@ +package alerting + +import ( + "testing" + + "reflect" + + "github.com/grafana/grafana/pkg/components/simplejson" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestAlertNotificationExtraction(t *testing.T) { + + Convey("Parsing alert notification from settings", t, func() { + Convey("Parsing email notification from settings", func() { + json := ` + { + "from": "alerting@grafana.org", + "to": "ops@grafana.org" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "email", + Settings: settingsJSON, + } + + not, err := NewNotificationFromDBModel(model) + + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "ops") + So(not.Type, ShouldEqual, "email") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier") + + email := not.Notifierr.(*EmailNotifier) + So(email.To, ShouldEqual, "ops@grafana.org") + So(email.From, ShouldEqual, "alerting@grafana.org") + }) + + Convey("Parsing webhook notification from settings", func() { + json := ` + { + "url": "http://localhost:3000", + "username": "username", + "password": "password" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "slack", + Type: "webhook", + Settings: settingsJSON, + } + + not, err := NewNotificationFromDBModel(model) + + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "slack") + So(not.Type, ShouldEqual, "webhook") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier") + + webhook := not.Notifierr.(*WebhookNotifier) + So(webhook.Url, ShouldEqual, "http://localhost:3000") + }) + }) +} diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index de2222c995f..7694c7bf0e0 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -3,6 +3,7 @@ package sqlstore import ( "bytes" "fmt" + "strconv" "time" "github.com/go-xorm/xorm" @@ -43,6 +44,25 @@ func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Sessio params = append(params, query.Name) } + if query.Id != 0 { + sql.WriteString(` AND alert_notification.id = ?`) + params = append(params, strconv.Itoa(int(query.Id))) + } + + if len(query.Ids) > 0 { + sql.WriteString(` AND (`) + + for i, id := range query.Ids { + if i != 0 { + sql.WriteString(` OR`) + } + sql.WriteString(` alert_notification.id = ?`) + params = append(params, id) + } + + sql.WriteString(`)`) + } + var result []*m.AlertNotification if err := sess.Sql(sql.String(), params...).Find(&result); err != nil { return err @@ -71,15 +91,16 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error Type: cmd.Type, Created: time.Now(), Settings: cmd.Settings, + Updated: time.Now(), } - id, err := sess.Insert(alertNotification) + _, err = sess.Insert(alertNotification) if err != nil { return err } - alertNotification.Id = id + //alertNotification.Id = int(id) cmd.Result = alertNotification return nil }) @@ -87,30 +108,34 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return inTransaction(func(sess *xorm.Session) (err error) { - alertNotification := &m.AlertNotification{} - - var has bool - has, err = sess.Id(cmd.Id).Get(alertNotification) + current := &m.AlertNotification{} + _, err = sess.Id(cmd.Id).Get(current) if err != nil { return err } - if !has { - return fmt.Errorf("Alert notification does not exist") - } - + alertNotification := &m.AlertNotification{} + alertNotification.Id = cmd.Id + alertNotification.OrgId = cmd.OrgID alertNotification.Name = cmd.Name alertNotification.Type = cmd.Type alertNotification.Settings = cmd.Settings alertNotification.Updated = time.Now() + alertNotification.Created = current.Created - _, err = sess.Id(alertNotification.Id).Cols("name", "type", "settings", "updated").Update(alertNotification) + var affected int64 + //affected, err = sess.Id(alertNotification.Id).Cols("name", "type", "settings", "updated").Update(alertNotification) + affected, err = sess.Id(alertNotification.Id).Update(alertNotification) if err != nil { return err } + if affected == 0 { + return fmt.Errorf("Could not find alert notification") + } + cmd.Result = alertNotification return nil }) diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 48dcd23b751..0b5b8e8cf13 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) @@ -27,41 +28,62 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", + Name: "ops", + Type: "email", + OrgID: 1, + Settings: simplejson.New(), } err = CreateAlertNotificationCommand(cmd) So(err, ShouldBeNil) So(cmd.Result.Id, ShouldNotEqual, 0) + So(cmd.Result.OrgId, ShouldNotEqual, 0) + So(cmd.Result.Type, ShouldEqual, "email") Convey("Cannot save Alert Notification with the same name", func() { err = CreateAlertNotificationCommand(cmd) So(err, ShouldNotBeNil) }) - Convey("Cannot update alert notification that does not exist", func() { - newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: cmd.Result.Type, - OrgID: cmd.Result.OrgId, - Id: 1337, - } - err = UpdateAlertNotification(newCmd) - So(err, ShouldNotBeNil) - }) - Convey("Can update alert notification", func() { newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: cmd.Result.Type, - OrgID: cmd.Result.OrgId, - Id: cmd.Result.Id, + Name: "NewName", + Type: "webhook", + OrgID: cmd.Result.OrgId, + Settings: simplejson.New(), + Id: cmd.Result.Id, } - err = UpdateAlertNotification(newCmd) + err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) So(newCmd.Result.Name, ShouldEqual, "NewName") }) }) + + Convey("Can search using an array of ids", func() { + So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{ + Name: "ops2", + Type: "email", + OrgID: 1, + Settings: simplejson.New(), + }), ShouldBeNil) + + So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{ + Name: "slack", + Type: "webhook", + OrgID: 1, + Settings: simplejson.New(), + }), ShouldBeNil) + + Convey("search", func() { + query := &m.GetAlertNotificationQuery{ + Ids: []int64{1, 2, 3}, + OrgID: 1, + } + + err := AlertNotificationQuery(query) + So(err, ShouldBeNil) + So(len(query.Result), ShouldEqual, 2) + }) + }) }) } From b9b65cf2d416068bcd2cb25a76298b7e9560536c Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 15 Jun 2016 09:19:22 +0200 Subject: [PATCH 05/41] tech(alerting): add logging about failed notifications --- pkg/models/alert.go | 4 +++ pkg/services/alerting/engine.go | 26 +++++++++++----- pkg/services/alerting/interfaces.go | 4 +++ pkg/services/alerting/notifier.go | 48 +++++++++++++++++------------ 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 148a85cc7c5..64562534e9e 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -27,6 +27,10 @@ func (alert *Alert) ValidToSave() bool { return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0 } +func (alert *Alert) ShouldUpdateState(newState string) bool { + return alert.State != newState +} + func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 06d323ef14f..458fabd2f6b 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -20,6 +20,7 @@ type Engine struct { handler AlertingHandler ruleReader RuleReader log log.Logger + notifier Notifier } func NewEngine() *Engine { @@ -31,6 +32,7 @@ func NewEngine() *Engine { handler: NewHandler(), ruleReader: NewRuleReader(), log: log.New("alerting.engine"), + notifier: NewNotifier(), } return e @@ -129,13 +131,23 @@ func (e *Engine) resultHandler() { } func (e *Engine) saveState(result *AlertResult) { - cmd := &m.UpdateAlertStateCommand{ - AlertId: result.AlertJob.Rule.Id, - NewState: result.State, - Info: result.Description, - } + query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id} + bus.Dispatch(query) - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to save state", "error", err) + if query.Result.ShouldUpdateState(result.State) { + cmd := &m.UpdateAlertStateCommand{ + AlertId: result.AlertJob.Rule.Id, + NewState: result.State, + Info: result.Description, + } + + if err := bus.Dispatch(cmd); err != nil { + e.log.Error("Failed to save state", "error", err) + } + + e.log.Debug("will notify! about", "new state", result.State) + e.notifier.Notify(result) + } else { + e.log.Debug("state remains the same!") } } diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index e57ed62a9a6..87ded1a2631 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -10,3 +10,7 @@ type Scheduler interface { Tick(time time.Time, execQueue chan *AlertJob) Update(rules []*AlertRule) } + +type Notifier interface { + Notify(alertResult *AlertResult) +} diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index a843a19b0eb..6dbd4812190 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1,29 +1,32 @@ package alerting import ( + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" ) -type Notifier struct { +type NotifierImpl struct { log log.Logger } -func NewNotifier() *Notifier { - return &Notifier{ +func NewNotifier() *NotifierImpl { + return &NotifierImpl{ log: log.New("alerting.notifier"), } } -func (n *Notifier) Notify(alertResult AlertResult) { - notifiers := getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) +func (n *NotifierImpl) Notify(alertResult *AlertResult) { + n.log.Warn("LETS NOTIFY!!!!A") + notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, []int64{1, 2}) for _, notifier := range notifiers { + warn := alertResult.State == alertstates.Warn && notifier.SendWarning crit := alertResult.State == alertstates.Critical && notifier.SendCritical - + n.log.Warn("looopie", "warn", warn, "crit", crit) if warn || crit { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) go notifier.Notifierr.Notify(alertResult) @@ -44,41 +47,44 @@ type Notification struct { type EmailNotifier struct { To string From string + log log.Logger } -func (this EmailNotifier) Notify(alertResult AlertResult) { +func (this *EmailNotifier) Notify(alertResult *AlertResult) { //bus.dispath to notification package in grafana + this.log.Info("Sending email") } type WebhookNotifier struct { Url string AuthUser string AuthPassword string + log log.Logger } -func (this WebhookNotifier) Notify(alertResult AlertResult) { +func (this *WebhookNotifier) Notify(alertResult *AlertResult) { //bus.dispath to notification package in grafana + this.log.Info("Sending webhook") } type Notifierr interface { - Notify(alertResult AlertResult) + Notify(alertResult *AlertResult) } -func getNotifiers(orgId int64, notificationGroups []int64) []*Notification { - var notifications []*m.AlertNotification - - for _, notificationId := range notificationGroups { - query := m.GetAlertNotificationQuery{ - OrgID: orgId, - Id: notificationId, - } - - notifications = append(notifications, query.Result...) +func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification { + query := &m.GetAlertNotificationQuery{ + OrgID: orgId, + Ids: notificationGroups, + } + err := bus.Dispatch(query) + if err != nil { + n.log.Error("Failed to read notifications", "error", err) } var result []*Notification - for _, notification := range notifications { + n.log.Warn("query result", "length", len(query.Result)) + for _, notification := range query.Result { not, err := NewNotificationFromDBModel(notification) if err == nil { result = append(result, not) @@ -103,6 +109,7 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No return &EmailNotifier{ To: settings.Get("to").MustString(), From: settings.Get("from").MustString(), + log: log.New("alerting.notification.email"), } } @@ -110,5 +117,6 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No Url: settings.Get("url").MustString(), AuthUser: settings.Get("user").MustString(), AuthPassword: settings.Get("password").MustString(), + log: log.New("alerting.notification.webhook"), } } From 4c5d2d60792e6f26850ef0d4da23af4334aab849 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 15 Jun 2016 09:20:26 +0200 Subject: [PATCH 06/41] style(alerting): remove unused code --- pkg/services/alerting/datasources/backends.go | 3 - pkg/services/alerting/datasources/graphite.go | 80 ------------------- 2 files changed, 83 deletions(-) delete mode 100644 pkg/services/alerting/datasources/backends.go delete mode 100644 pkg/services/alerting/datasources/graphite.go diff --git a/pkg/services/alerting/datasources/backends.go b/pkg/services/alerting/datasources/backends.go deleted file mode 100644 index 95ca132d85a..00000000000 --- a/pkg/services/alerting/datasources/backends.go +++ /dev/null @@ -1,3 +0,0 @@ -package datasources - -// GetSeries returns timeseries data from the datasource diff --git a/pkg/services/alerting/datasources/graphite.go b/pkg/services/alerting/datasources/graphite.go deleted file mode 100644 index 73309ca3b66..00000000000 --- a/pkg/services/alerting/datasources/graphite.go +++ /dev/null @@ -1,80 +0,0 @@ -package datasources - -// import ( -// "bytes" -// "encoding/json" -// "fmt" -// "io/ioutil" -// "net/http" -// "net/url" -// "strconv" -// "time" -// -// "github.com/grafana/grafana/pkg/components/simplejson" -// "github.com/grafana/grafana/pkg/log" -// m "github.com/grafana/grafana/pkg/models" -// "github.com/grafana/grafana/pkg/util" -// ) -// -// type GraphiteClient struct{} -// -// type GraphiteSerie struct { -// Datapoints [][2]float64 -// Target string -// } -// -// var DefaultClient = &http.Client{ -// Timeout: time.Minute, -// } -// -// type GraphiteResponse []GraphiteSerie -// -// func (client GraphiteClient) GetSeries(rule m.AlertJob, datasource m.DataSource) (m.TimeSeriesSlice, error) { -// v := url.Values{ -// "format": []string{"json"}, -// "target": []string{getTargetFromRule(rule.Rule)}, -// "until": []string{"now"}, -// "from": []string{"-" + strconv.Itoa(rule.Rule.QueryRange) + "s"}, -// } -// -// log.Trace("Graphite: sending request with querystring: ", v.Encode()) -// -// req, err := http.NewRequest("POST", datasource.Url+"/render", nil) -// -// if err != nil { -// return nil, fmt.Errorf("Could not create request") -// } -// -// req.Body = ioutil.NopCloser(bytes.NewReader([]byte(v.Encode()))) -// -// if datasource.BasicAuth { -// req.Header.Add("Authorization", util.GetBasicAuthHeader(datasource.User, datasource.Password)) -// } -// -// res, err := DefaultClient.Do(req) -// -// if err != nil { -// return nil, err -// } -// -// if res.StatusCode != http.StatusOK { -// return nil, fmt.Errorf("expected httpstatus 200, found %d", res.StatusCode) -// } -// -// response := GraphiteResponse{} -// -// json.NewDecoder(res.Body).Decode(&response) -// -// var timeSeries []*m.TimeSeries -// for _, v := range response { -// timeSeries = append(timeSeries, m.NewTimeSeries(v.Target, v.Datapoints)) -// } -// -// return timeSeries, nil -// } -// -// func getTargetFromRule(rule m.AlertRule) string { -// json, _ := simplejson.NewJson([]byte(rule.Query)) -// -// return json.Get("target").MustString() -// } From a3b7ea77040ca199bf3a70b17f4e3484017891de Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 15 Jun 2016 10:48:04 +0200 Subject: [PATCH 07/41] feat(alerting): skeleton for alert notification configuration page --- pkg/api/alerting.go | 32 +++++++++++++++++++ pkg/api/api.go | 7 ++++ pkg/services/alerting/notifier.go | 14 ++++---- public/app/core/routes/routes.ts | 6 ++++ .../alerting/alert_notifications_ctrl.ts | 20 ++++++++++++ public/app/features/alerting/all.ts | 1 + .../partials/alert_notifications.html | 20 ++++++++++++ 7 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 public/app/features/alerting/alert_notifications_ctrl.ts create mode 100644 public/app/features/alerting/partials/alert_notifications.html diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 3f2c38486a4..e54f8adf0c4 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -156,3 +156,35 @@ func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Re return Json(200, cmd.Result) } + +func GetAlertNotifications(c *middleware.Context) Response { + query := &models.GetAlertNotificationQuery{ + OrgID: c.OrgId, + } + + if err := bus.Dispatch(query); err != nil { + return ApiError(500, "Failed to get alert notifications", err) + } + + return Json(200, query.Result) +} + +func CreateAlertNotification(c *middleware.Context, cmd *models.CreateAlertNotificationCommand) Response { + cmd.OrgID = c.OrgId + + if err := bus.Dispatch(cmd); err != nil { + return ApiError(500, "Failed to create alert notification", err) + } + + return Json(200, cmd.Result) +} + +func UpdateAlertNotification(c *middleware.Context, cmd *models.UpdateAlertNotificationCommand) Response { + cmd.OrgID = c.OrgId + + if err := bus.Dispatch(cmd); err != nil { + return ApiError(500, "Failed to update alert notification", err) + } + + return Json(200, cmd.Result) +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 4483f64530f..5de22c7ba51 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -59,6 +59,7 @@ func Register(r *macaron.Macaron) { r.Get("/playlists/", reqSignedIn, Index) r.Get("/playlists/*", reqSignedIn, Index) r.Get("/alerting/", reqSignedIn, Index) + r.Get("/alerting/*", reqSignedIn, Index) // sign up r.Get("/signup", Index) @@ -250,6 +251,12 @@ func Register(r *macaron.Macaron) { r.Get("/", wrap(GetAlerts)) }) + r.Get("/notifications", wrap(GetAlertNotifications)) + r.Group("/notification", func() { + r.Post("/", bind(m.CreateAlertNotificationCommand{}), wrap(CreateAlertNotification)) + r.Put("/", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) + }) + r.Get("/changes", wrap(GetAlertChanges)) }) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 6dbd4812190..48fa4767437 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -29,7 +29,7 @@ func (n *NotifierImpl) Notify(alertResult *AlertResult) { n.log.Warn("looopie", "warn", warn, "crit", crit) if warn || crit { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) - go notifier.Notifierr.Notify(alertResult) + go notifier.Notifierr.Dispatch(alertResult) } } @@ -41,7 +41,7 @@ type Notification struct { SendWarning bool SendCritical bool - Notifierr Notifierr + Notifierr NotificationDispatcher } type EmailNotifier struct { @@ -50,7 +50,7 @@ type EmailNotifier struct { log log.Logger } -func (this *EmailNotifier) Notify(alertResult *AlertResult) { +func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { //bus.dispath to notification package in grafana this.log.Info("Sending email") } @@ -62,13 +62,13 @@ type WebhookNotifier struct { log log.Logger } -func (this *WebhookNotifier) Notify(alertResult *AlertResult) { +func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) { //bus.dispath to notification package in grafana this.log.Info("Sending webhook") } -type Notifierr interface { - Notify(alertResult *AlertResult) +type NotificationDispatcher interface { + Dispatch(alertResult *AlertResult) } func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification { @@ -104,7 +104,7 @@ func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, erro }, nil } -var createNotifier = func(notificationType string, settings *simplejson.Json) Notifierr { +var createNotifier = func(notificationType string, settings *simplejson.Json) NotificationDispatcher { if notificationType == "email" { return &EmailNotifier{ To: settings.Get("to").MustString(), diff --git a/public/app/core/routes/routes.ts b/public/app/core/routes/routes.ts index a1655f23bd2..1dccb5a1ea0 100644 --- a/public/app/core/routes/routes.ts +++ b/public/app/core/routes/routes.ts @@ -201,6 +201,12 @@ function setupAngularRoutes($routeProvider, $locationProvider) { controllerAs: 'ctrl', resolve: loadAlertingBundle, }) + .when('/alerting/notifications', { + templateUrl: 'public/app/features/alerting/partials/alert_notifications.html', + controller: 'AlertNotificationsCtrl', + contrllerAs: 'ctrl', + resolve: loadAlertingBundle, + }) .when('/alerting/:alertId/states', { templateUrl: 'public/app/features/alerting/partials/alert_log.html', controller: 'AlertLogCtrl', diff --git a/public/app/features/alerting/alert_notifications_ctrl.ts b/public/app/features/alerting/alert_notifications_ctrl.ts new file mode 100644 index 00000000000..ac3ad532450 --- /dev/null +++ b/public/app/features/alerting/alert_notifications_ctrl.ts @@ -0,0 +1,20 @@ +/// + +import angular from 'angular'; +import _ from 'lodash'; +import coreModule from '../../core/core_module'; +import config from 'app/core/config'; + +export class AlertNotificationsCtrl { + + /** @ngInject */ + constructor(private backendSrv) { + this.loadNotifications(); + } + + loadNotifications() { + } +} + +coreModule.controller('AlertNotificationsCtrl', AlertNotificationsCtrl); + diff --git a/public/app/features/alerting/all.ts b/public/app/features/alerting/all.ts index 9ac8dcafb9b..5384a5d97f8 100644 --- a/public/app/features/alerting/all.ts +++ b/public/app/features/alerting/all.ts @@ -1,3 +1,4 @@ import './alerts_ctrl'; import './alert_log_ctrl'; +import './alert_notifications_ctrl'; diff --git a/public/app/features/alerting/partials/alert_notifications.html b/public/app/features/alerting/partials/alert_notifications.html new file mode 100644 index 00000000000..0b0ed616fa4 --- /dev/null +++ b/public/app/features/alerting/partials/alert_notifications.html @@ -0,0 +1,20 @@ + + + + +
+
+

Alert notifications

+
+ + + + + + + + +
Name
+ Name +
+
From efea3bc9cbf4f0a6a8592e458e422f5dc42b2add Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 15 Jun 2016 14:45:05 +0200 Subject: [PATCH 08/41] feat(alerting): skeleton commit for webhook --- pkg/models/{emails.go => notifications.go} | 8 +++ pkg/services/alerting/engine.go | 3 +- pkg/services/alerting/notifier.go | 23 +++++++- pkg/services/notifications/notifications.go | 15 +++++ pkg/services/notifications/webhook.go | 62 +++++++++++++++++++++ 5 files changed, 107 insertions(+), 4 deletions(-) rename pkg/models/{emails.go => notifications.go} (73%) create mode 100644 pkg/services/notifications/webhook.go diff --git a/pkg/models/emails.go b/pkg/models/notifications.go similarity index 73% rename from pkg/models/emails.go rename to pkg/models/notifications.go index 74da180f7d8..cd62a4c046d 100644 --- a/pkg/models/emails.go +++ b/pkg/models/notifications.go @@ -12,6 +12,14 @@ type SendEmailCommand struct { Info string } +type SendWebhook struct { + Url string + AuthUser string + AuthPassword string + Body string + Method string +} + type SendResetPasswordEmailCommand struct { User *User } diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 458fabd2f6b..88d39929efe 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -134,6 +134,7 @@ func (e *Engine) saveState(result *AlertResult) { query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id} bus.Dispatch(query) + e.notifier.Notify(result) if query.Result.ShouldUpdateState(result.State) { cmd := &m.UpdateAlertStateCommand{ AlertId: result.AlertJob.Rule.Id, @@ -146,7 +147,7 @@ func (e *Engine) saveState(result *AlertResult) { } e.log.Debug("will notify! about", "new state", result.State) - e.notifier.Notify(result) + } else { e.log.Debug("state remains the same!") } diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 48fa4767437..7378ff45fda 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -51,20 +51,36 @@ type EmailNotifier struct { } func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { - //bus.dispath to notification package in grafana - this.log.Info("Sending email") + /* + this.log.Info("Sending email") + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{}, + To: []string{}, + Info: "", + Massive: false, + Template: "", + } + + bus.Dispatch(cmd) + */ } type WebhookNotifier struct { Url string + Method string AuthUser string AuthPassword string log log.Logger } func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) { - //bus.dispath to notification package in grafana this.log.Info("Sending webhook") + cmd := &m.SendWebhook{ + Url: this.Url, + Method: this.Method, + } + + bus.Dispatch(cmd) } type NotificationDispatcher interface { @@ -115,6 +131,7 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No return &WebhookNotifier{ Url: settings.Get("url").MustString(), + Method: settings.Get("method").MustString(), AuthUser: settings.Get("user").MustString(), AuthPassword: settings.Get("password").MustString(), log: log.New("alerting.notification.webhook"), diff --git a/pkg/services/notifications/notifications.go b/pkg/services/notifications/notifications.go index 63ce7219618..a22acbc2d36 100644 --- a/pkg/services/notifications/notifications.go +++ b/pkg/services/notifications/notifications.go @@ -23,11 +23,14 @@ var tmplWelcomeOnSignUp = "welcome_on_signup.html" func Init() error { initMailQueue() + initWebhookQueue() bus.AddHandler("email", sendResetPasswordEmail) bus.AddHandler("email", validateResetPasswordCode) bus.AddHandler("email", sendEmailCommandHandler) + bus.AddHandler("webhook", sendWebhook) + bus.AddEventListener(signUpStartedHandler) bus.AddEventListener(signUpCompletedHandler) @@ -53,6 +56,18 @@ func Init() error { return nil } +func sendWebhook(cmd *m.SendWebhook) error { + addToWebhookQueue(&Webhook{ + Url: cmd.Url, + AuthUser: cmd.AuthUser, + AuthPassword: cmd.AuthPassword, + Method: cmd.Method, + Body: cmd.Body, + }) + + return nil +} + func subjectTemplateFunc(obj map[string]interface{}, value string) string { obj["value"] = value return "" diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go new file mode 100644 index 00000000000..7d591ddb26b --- /dev/null +++ b/pkg/services/notifications/webhook.go @@ -0,0 +1,62 @@ +package notifications + +import ( + "net/http" + "time" + + "github.com/grafana/grafana/pkg/log" +) + +type Webhook struct { + Url string + AuthUser string + AuthPassword string + Body string + Method string +} + +var webhookQueue chan *Webhook +var webhookLog log.Logger + +func initWebhookQueue() { + webhookLog = log.New("notifications.webhook") + webhookQueue = make(chan *Webhook, 10) + go processWebhookQueue() +} + +func processWebhookQueue() { + for { + select { + case webhook := <-webhookQueue: + err := sendWebRequest(webhook) + + if err != nil { + webhookLog.Error("Failed to send webrequest ") + } + } + } +} + +func sendWebRequest(webhook *Webhook) error { + webhookLog.Error("Sending stuff! ", "url", webhook.Url) + + client := http.Client{Timeout: time.Duration(3 * time.Second)} + + request, err := http.NewRequest(webhook.Method, webhook.Url, nil /*io.reader*/) + + if err != nil { + return err + } + + resp, err := client.Do(request) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +var addToWebhookQueue = func(msg *Webhook) { + webhookQueue <- msg +} From 2e809cae057ccb82f569408742203eda7e72e8d6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 08:15:48 +0200 Subject: [PATCH 09/41] tech(alerting): enforce POST for webhooks --- pkg/models/notifications.go | 9 ++++---- pkg/services/alerting/notifier.go | 24 ++++++++++----------- pkg/services/notifications/notifications.go | 9 ++++---- pkg/services/notifications/webhook.go | 20 +++++++++++------ 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/pkg/models/notifications.go b/pkg/models/notifications.go index cd62a4c046d..d357b9cf562 100644 --- a/pkg/models/notifications.go +++ b/pkg/models/notifications.go @@ -13,11 +13,10 @@ type SendEmailCommand struct { } type SendWebhook struct { - Url string - AuthUser string - AuthPassword string - Body string - Method string + Url string + User string + Password string + Body string } type SendResetPasswordEmailCommand struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 7378ff45fda..ec48a8fbcb7 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -66,18 +66,19 @@ func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { } type WebhookNotifier struct { - Url string - Method string - AuthUser string - AuthPassword string - log log.Logger + Url string + User string + Password string + log log.Logger } func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) { this.log.Info("Sending webhook") cmd := &m.SendWebhook{ - Url: this.Url, - Method: this.Method, + Url: this.Url, + User: this.User, + Password: this.Password, + Body: alertResult.Description, } bus.Dispatch(cmd) @@ -130,10 +131,9 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No } return &WebhookNotifier{ - Url: settings.Get("url").MustString(), - Method: settings.Get("method").MustString(), - AuthUser: settings.Get("user").MustString(), - AuthPassword: settings.Get("password").MustString(), - log: log.New("alerting.notification.webhook"), + Url: settings.Get("url").MustString(), + User: settings.Get("user").MustString(), + Password: settings.Get("password").MustString(), + log: log.New("alerting.notification.webhook"), } } diff --git a/pkg/services/notifications/notifications.go b/pkg/services/notifications/notifications.go index a22acbc2d36..04b11f73b84 100644 --- a/pkg/services/notifications/notifications.go +++ b/pkg/services/notifications/notifications.go @@ -58,11 +58,10 @@ func Init() error { func sendWebhook(cmd *m.SendWebhook) error { addToWebhookQueue(&Webhook{ - Url: cmd.Url, - AuthUser: cmd.AuthUser, - AuthPassword: cmd.AuthPassword, - Method: cmd.Method, - Body: cmd.Body, + Url: cmd.Url, + User: cmd.User, + Password: cmd.Password, + Body: cmd.Body, }) return nil diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go index 7d591ddb26b..11a8839fbc8 100644 --- a/pkg/services/notifications/webhook.go +++ b/pkg/services/notifications/webhook.go @@ -1,18 +1,19 @@ package notifications import ( + "bytes" "net/http" "time" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/util" ) type Webhook struct { - Url string - AuthUser string - AuthPassword string - Body string - Method string + Url string + User string + Password string + Body string } var webhookQueue chan *Webhook @@ -40,9 +41,14 @@ func processWebhookQueue() { func sendWebRequest(webhook *Webhook) error { webhookLog.Error("Sending stuff! ", "url", webhook.Url) - client := http.Client{Timeout: time.Duration(3 * time.Second)} + client := http.Client{ + Timeout: time.Duration(3 * time.Second), + } - request, err := http.NewRequest(webhook.Method, webhook.Url, nil /*io.reader*/) + request, err := http.NewRequest("POST", webhook.Url, bytes.NewReader([]byte(webhook.Body))) + if webhook.User != "" && webhook.Password != "" { + request.Header.Add("Authorization", util.GetBasicAuthHeader(webhook.User, webhook.Password)) + } if err != nil { return err From 4d03e0417213a77818c64ddd74564f1469e678d3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 08:29:49 +0200 Subject: [PATCH 10/41] feat(alerting): enable email notifiter --- pkg/services/alerting/notifier.go | 33 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index ec48a8fbcb7..b2374b2c125 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -45,24 +45,24 @@ type Notification struct { } type EmailNotifier struct { - To string - From string - log log.Logger + To string + log log.Logger } func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { - /* - this.log.Info("Sending email") - cmd := &m.SendEmailCommand{ - Data: map[string]interface{}{}, - To: []string{}, - Info: "", - Massive: false, - Template: "", - } + this.log.Info("Sending email") + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{ + "Description": alertResult.Description, + "TriggeredAlerts": alertResult.TriggeredAlerts, + }, + To: []string{this.To}, + Info: "Alert result", + Massive: false, + Template: "", + } - bus.Dispatch(cmd) - */ + bus.Dispatch(cmd) } type WebhookNotifier struct { @@ -124,9 +124,8 @@ func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, erro var createNotifier = func(notificationType string, settings *simplejson.Json) NotificationDispatcher { if notificationType == "email" { return &EmailNotifier{ - To: settings.Get("to").MustString(), - From: settings.Get("from").MustString(), - log: log.New("alerting.notification.email"), + To: settings.Get("to").MustString(), + log: log.New("alerting.notification.email"), } } From 72e23bca5f23c3616ca709fecefd0b498bc94131 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 08:56:19 +0200 Subject: [PATCH 11/41] tech(alerting): improve/cleanup logging --- pkg/services/alerting/notifier.go | 4 ---- pkg/services/notifications/webhook.go | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index b2374b2c125..6e31fed7db3 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -19,14 +19,11 @@ func NewNotifier() *NotifierImpl { } func (n *NotifierImpl) Notify(alertResult *AlertResult) { - n.log.Warn("LETS NOTIFY!!!!A") notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, []int64{1, 2}) for _, notifier := range notifiers { - warn := alertResult.State == alertstates.Warn && notifier.SendWarning crit := alertResult.State == alertstates.Critical && notifier.SendCritical - n.log.Warn("looopie", "warn", warn, "crit", crit) if warn || crit { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) go notifier.Notifierr.Dispatch(alertResult) @@ -100,7 +97,6 @@ func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []* var result []*Notification - n.log.Warn("query result", "length", len(query.Result)) for _, notification := range query.Result { not, err := NewNotificationFromDBModel(notification) if err == nil { diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go index 11a8839fbc8..1696d771387 100644 --- a/pkg/services/notifications/webhook.go +++ b/pkg/services/notifications/webhook.go @@ -32,15 +32,13 @@ func processWebhookQueue() { err := sendWebRequest(webhook) if err != nil { - webhookLog.Error("Failed to send webrequest ") + webhookLog.Error("Failed to send webrequest ", "error", err) } } } } func sendWebRequest(webhook *Webhook) error { - webhookLog.Error("Sending stuff! ", "url", webhook.Url) - client := http.Client{ Timeout: time.Duration(3 * time.Second), } From 00fc2e259327804df6f562f60fdf11033afaa221 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 09:09:09 +0200 Subject: [PATCH 12/41] test(alerting): fixes broken unittest --- pkg/services/alerting/notifier_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index 23af5eb41cf..4249238cfa0 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -16,7 +16,6 @@ func TestAlertNotificationExtraction(t *testing.T) { Convey("Parsing email notification from settings", func() { json := ` { - "from": "alerting@grafana.org", "to": "ops@grafana.org" }` @@ -36,7 +35,6 @@ func TestAlertNotificationExtraction(t *testing.T) { email := not.Notifierr.(*EmailNotifier) So(email.To, ShouldEqual, "ops@grafana.org") - So(email.From, ShouldEqual, "alerting@grafana.org") }) Convey("Parsing webhook notification from settings", func() { From 7f767224afcc8dca20df24b0c04523b4e9451615 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 14:29:20 +0200 Subject: [PATCH 13/41] feat(alerting): basic support for creating and updating notifications --- pkg/api/alerting.go | 21 ++++++-- pkg/api/api.go | 4 +- pkg/models/alert_notifications.go | 33 ++++++------ public/app/core/routes/routes.ts | 18 +++++-- public/app/features/alerting/all.ts | 3 +- .../alerting/notification_edit_ctrl.ts | 49 ++++++++++++++++++ ...ons_ctrl.ts => notifications_list_ctrl.ts} | 9 +++- .../partials/alert_notifications.html | 20 -------- .../alerting/partials/notification_edit.html | 51 +++++++++++++++++++ .../alerting/partials/notifications_list.html | 37 ++++++++++++++ 10 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 public/app/features/alerting/notification_edit_ctrl.ts rename public/app/features/alerting/{alert_notifications_ctrl.ts => notifications_list_ctrl.ts} (55%) delete mode 100644 public/app/features/alerting/partials/alert_notifications.html create mode 100644 public/app/features/alerting/partials/notification_edit.html create mode 100644 public/app/features/alerting/partials/notifications_list.html diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index e54f8adf0c4..514a8498056 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -169,20 +169,33 @@ func GetAlertNotifications(c *middleware.Context) Response { return Json(200, query.Result) } -func CreateAlertNotification(c *middleware.Context, cmd *models.CreateAlertNotificationCommand) Response { +func GetAlertNotificationById(c *middleware.Context) Response { + query := &models.GetAlertNotificationQuery{ + OrgID: c.OrgId, + Id: c.ParamsInt64("notificationId"), + } + + if err := bus.Dispatch(query); err != nil { + return ApiError(500, "Failed to get alert notifications", err) + } + + return Json(200, query.Result[0]) +} + +func CreateAlertNotification(c *middleware.Context, cmd models.CreateAlertNotificationCommand) Response { cmd.OrgID = c.OrgId - if err := bus.Dispatch(cmd); err != nil { + if err := bus.Dispatch(&cmd); err != nil { return ApiError(500, "Failed to create alert notification", err) } return Json(200, cmd.Result) } -func UpdateAlertNotification(c *middleware.Context, cmd *models.UpdateAlertNotificationCommand) Response { +func UpdateAlertNotification(c *middleware.Context, cmd models.UpdateAlertNotificationCommand) Response { cmd.OrgID = c.OrgId - if err := bus.Dispatch(cmd); err != nil { + if err := bus.Dispatch(&cmd); err != nil { return ApiError(500, "Failed to update alert notification", err) } diff --git a/pkg/api/api.go b/pkg/api/api.go index 5de22c7ba51..706a46c0ec7 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -252,9 +252,11 @@ func Register(r *macaron.Macaron) { }) r.Get("/notifications", wrap(GetAlertNotifications)) + r.Group("/notification", func() { r.Post("/", bind(m.CreateAlertNotificationCommand{}), wrap(CreateAlertNotification)) - r.Put("/", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) + r.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) + r.Get("/:notificationId", wrap(GetAlertNotificationById)) }) r.Get("/changes", wrap(GetAlertChanges)) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 7c22351b172..55318b9b285 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -7,31 +7,30 @@ import ( ) type AlertNotification struct { - Id int64 - OrgId int64 - Name string - Type string - Settings *simplejson.Json - - Created time.Time - Updated time.Time + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string - Type string - OrgID int64 - Settings *simplejson.Json + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + OrgID int64 `json:"-"` + Settings *simplejson.Json `json:"settings"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 - Name string - Type string - OrgID int64 - Settings *simplejson.Json + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + OrgID int64 `json:"-"` + Settings *simplejson.Json `json:"settings" binding:"Required"` Result *AlertNotification } diff --git a/public/app/core/routes/routes.ts b/public/app/core/routes/routes.ts index 1dccb5a1ea0..5f83013433b 100644 --- a/public/app/core/routes/routes.ts +++ b/public/app/core/routes/routes.ts @@ -202,9 +202,21 @@ function setupAngularRoutes($routeProvider, $locationProvider) { resolve: loadAlertingBundle, }) .when('/alerting/notifications', { - templateUrl: 'public/app/features/alerting/partials/alert_notifications.html', - controller: 'AlertNotificationsCtrl', - contrllerAs: 'ctrl', + templateUrl: 'public/app/features/alerting/partials/notifications_list.html', + controller: 'AlertNotificationsListCtrl', + controllerAs: 'ctrl', + resolve: loadAlertingBundle, + }) + .when('/alerting/notification/new', { + templateUrl: 'public/app/features/alerting/partials/notification_edit.html', + controller: 'AlertNotificationEditCtrl', + controllerAs: 'ctrl', + resolve: loadAlertingBundle, + }) + .when('/alerting/notification/:notificationId/edit', { + templateUrl: 'public/app/features/alerting/partials/notification_edit.html', + controller: 'AlertNotificationEditCtrl', + controllerAs: 'ctrl', resolve: loadAlertingBundle, }) .when('/alerting/:alertId/states', { diff --git a/public/app/features/alerting/all.ts b/public/app/features/alerting/all.ts index 5384a5d97f8..c7e2264c1c8 100644 --- a/public/app/features/alerting/all.ts +++ b/public/app/features/alerting/all.ts @@ -1,4 +1,5 @@ import './alerts_ctrl'; import './alert_log_ctrl'; -import './alert_notifications_ctrl'; +import './notifications_list_ctrl'; +import './notification_edit_ctrl'; diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts new file mode 100644 index 00000000000..796d57aa2f8 --- /dev/null +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -0,0 +1,49 @@ +/// + +import angular from 'angular'; +import _ from 'lodash'; +import coreModule from '../../core/core_module'; +import config from 'app/core/config'; + +export class AlertNotificationEditCtrl { + + notification: any; + + /** @ngInject */ + constructor(private $routeParams, private backendSrv) { + if ($routeParams.notificationId) { + this.loadNotification($routeParams.notificationId); + } + } + + loadNotification(notificationId) { + this.backendSrv.get(`/api/alerts/notification/${notificationId}`).then(result => { + console.log(result); + this.notification = result; + }); + } + + isNew() { + return this.notification === undefined || this.notification.id === undefined; + } + + save() { + if (this.notification.id) { + console.log('this.notification: ', this.notification); + this.backendSrv.put(`/api/alerts/notification/${this.notification.id}`, this.notification) + .then(result => { + this.notification = result; + console.log('updated notification', result); + }); + } else { + this.backendSrv.post(`/api/alerts/notification`, this.notification) + .then(result => { + this.notification = result; + console.log('created new notification', result); + }); + } + } +} + +coreModule.controller('AlertNotificationEditCtrl', AlertNotificationEditCtrl); + diff --git a/public/app/features/alerting/alert_notifications_ctrl.ts b/public/app/features/alerting/notifications_list_ctrl.ts similarity index 55% rename from public/app/features/alerting/alert_notifications_ctrl.ts rename to public/app/features/alerting/notifications_list_ctrl.ts index ac3ad532450..41458a08577 100644 --- a/public/app/features/alerting/alert_notifications_ctrl.ts +++ b/public/app/features/alerting/notifications_list_ctrl.ts @@ -5,7 +5,9 @@ import _ from 'lodash'; import coreModule from '../../core/core_module'; import config from 'app/core/config'; -export class AlertNotificationsCtrl { +export class AlertNotificationsListCtrl { + + notifications: any; /** @ngInject */ constructor(private backendSrv) { @@ -13,8 +15,11 @@ export class AlertNotificationsCtrl { } loadNotifications() { + this.backendSrv.get(`/api/alerts/notifications`).then(result => { + this.notifications = result; + }); } } -coreModule.controller('AlertNotificationsCtrl', AlertNotificationsCtrl); +coreModule.controller('AlertNotificationsListCtrl', AlertNotificationsListCtrl); diff --git a/public/app/features/alerting/partials/alert_notifications.html b/public/app/features/alerting/partials/alert_notifications.html deleted file mode 100644 index 0b0ed616fa4..00000000000 --- a/public/app/features/alerting/partials/alert_notifications.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - -
-
-

Alert notifications

-
- - - - - - - - -
Name
- Name -
-
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html new file mode 100644 index 00000000000..3c7b80ab147 --- /dev/null +++ b/public/app/features/alerting/partials/notification_edit.html @@ -0,0 +1,51 @@ + + + +
+
+

Alert notification

+
+ +
+
+ Name + +
+
+ Type +
+ +
+
+
+
+
+ Url + +
+
+
+ Username + +
+
+ Password + +
+
+
+
+
+ To + +
+
+ +
+ +
+
diff --git a/public/app/features/alerting/partials/notifications_list.html b/public/app/features/alerting/partials/notifications_list.html new file mode 100644 index 00000000000..c3347018cd4 --- /dev/null +++ b/public/app/features/alerting/partials/notifications_list.html @@ -0,0 +1,37 @@ + + + +
+
+

Alert notifications

+ +
+ + + + + + + + + + + + +
NameType
+ + {{alert.name}} + + + {{notification.type}} + + + + edit + +
+ +
From 149c2ae91394042fa7f4416f0f71d6b564ecadfd Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 15:21:23 +0200 Subject: [PATCH 14/41] feat(alerting): add submenu for alerting --- pkg/api/index.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/api/index.go b/pkg/api/index.go index 9377b21a957..4cfdad5cca9 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -80,10 +80,16 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { }) if setting.AlertingEnabled && (c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR) { + alertChildNavs := []*dtos.NavLink{ + {Text: "Home", Url: setting.AppSubUrl + "/alerting"}, + {Text: "Notifications", Url: setting.AppSubUrl + "/alerting/notifications"}, + } + data.MainNavLinks = append(data.MainNavLinks, &dtos.NavLink{ - Text: "Alerting", - Icon: "icon-gf icon-gf-monitoring", - Url: setting.AppSubUrl + "/alerting", + Text: "Alerting", + Icon: "icon-gf icon-gf-monitoring", + Url: setting.AppSubUrl + "/alerting", + Children: alertChildNavs, }) } From b907ce341c6a40f800fbdd41df32be4a4d19d964 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 15:21:44 +0200 Subject: [PATCH 15/41] feat(alerting): enables deletes for alert notifications --- pkg/api/alerting.go | 13 +++++++++++++ pkg/api/api.go | 1 + pkg/models/alert_notifications.go | 5 +++++ pkg/services/sqlstore/alert_notification.go | 14 ++++++++++++++ .../features/alerting/notification_edit_ctrl.ts | 10 +++++++--- .../alerting/notifications_list_ctrl.ts | 15 ++++++++++++++- .../alerting/partials/notifications_list.html | 17 ++++++++++------- 7 files changed, 64 insertions(+), 11 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 514a8498056..869fb726410 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -201,3 +201,16 @@ func UpdateAlertNotification(c *middleware.Context, cmd models.UpdateAlertNotifi return Json(200, cmd.Result) } + +func DeleteAlertNotification(c *middleware.Context) Response { + cmd := models.DeleteAlertNotificationCommand{ + OrgId: c.OrgId, + Id: c.ParamsInt64("notificationId"), + } + + if err := bus.Dispatch(&cmd); err != nil { + return ApiError(500, "Failed to delete alert notification", err) + } + + return Json(200, map[string]interface{}{"notificationId": cmd.Id}) +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 706a46c0ec7..022edb9c74c 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -257,6 +257,7 @@ func Register(r *macaron.Macaron) { r.Post("/", bind(m.CreateAlertNotificationCommand{}), wrap(CreateAlertNotification)) r.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) r.Get("/:notificationId", wrap(GetAlertNotificationById)) + r.Delete("/:notificationId", wrap(DeleteAlertNotification)) }) r.Get("/changes", wrap(GetAlertChanges)) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 55318b9b285..85fdbafe9c4 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -35,6 +35,11 @@ type UpdateAlertNotificationCommand struct { Result *AlertNotification } +type DeleteAlertNotificationCommand struct { + Id int64 + OrgId int64 +} + type GetAlertNotificationQuery struct { Name string Id int64 diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 7694c7bf0e0..4c4bdb6e4c4 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -15,6 +15,20 @@ func init() { bus.AddHandler("sql", AlertNotificationQuery) bus.AddHandler("sql", CreateAlertNotificationCommand) bus.AddHandler("sql", UpdateAlertNotification) + bus.AddHandler("sql", DeleteAlertNotification) +} + +func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { + return inTransaction(func(sess *xorm.Session) error { + sql := "DELETE FROM alert_notification WHERE alert_notification.org_id = ? AND alert_notification.id = ?" + _, err := sess.Exec(sql, cmd.OrgId, cmd.Id) + + if err != nil { + return err + } + + return nil + }) } func AlertNotificationQuery(query *m.GetAlertNotificationQuery) error { diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 796d57aa2f8..08b4c41b361 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -10,7 +10,7 @@ export class AlertNotificationEditCtrl { notification: any; /** @ngInject */ - constructor(private $routeParams, private backendSrv) { + constructor(private $routeParams, private backendSrv, private $scope) { if ($routeParams.notificationId) { this.loadNotification($routeParams.notificationId); } @@ -33,13 +33,17 @@ export class AlertNotificationEditCtrl { this.backendSrv.put(`/api/alerts/notification/${this.notification.id}`, this.notification) .then(result => { this.notification = result; - console.log('updated notification', result); + this.$scope.appEvent('alert-success', ['Notification created!', '']); + }, () => { + this.$scope.appEvent('alert-error', ['Unable to create notification.', '']); }); } else { this.backendSrv.post(`/api/alerts/notification`, this.notification) .then(result => { this.notification = result; - console.log('created new notification', result); + this.$scope.appEvent('alert-success', ['Notification updated!', '']); + }, () => { + this.$scope.appEvent('alert-error', ['Unable to update notification.', '']); }); } } diff --git a/public/app/features/alerting/notifications_list_ctrl.ts b/public/app/features/alerting/notifications_list_ctrl.ts index 41458a08577..54362104b31 100644 --- a/public/app/features/alerting/notifications_list_ctrl.ts +++ b/public/app/features/alerting/notifications_list_ctrl.ts @@ -10,7 +10,7 @@ export class AlertNotificationsListCtrl { notifications: any; /** @ngInject */ - constructor(private backendSrv) { + constructor(private backendSrv, private $scope) { this.loadNotifications(); } @@ -19,7 +19,20 @@ export class AlertNotificationsListCtrl { this.notifications = result; }); } + + deleteNotification(notificationId) { + this.backendSrv.delete(`/api/alerts/notification/${notificationId}`) + .then(() => { + this.notifications = this.notifications.filter(notification => { + return notification.id !== notificationId; + }); + this.$scope.appEvent('alert-success', ['Notification deleted', '']); + }, () => { + this.$scope.appEvent('alert-error', ['Unable to delete notification', '']); + }); + } } coreModule.controller('AlertNotificationsListCtrl', AlertNotificationsListCtrl); + diff --git a/public/app/features/alerting/partials/notifications_list.html b/public/app/features/alerting/partials/notifications_list.html index c3347018cd4..be0a44fa0e0 100644 --- a/public/app/features/alerting/partials/notifications_list.html +++ b/public/app/features/alerting/partials/notifications_list.html @@ -4,32 +4,35 @@

Alert notifications

- +
- +
- + - -
NameTypeType
- {{alert.name}} + {{notification.name}} + {{notification.type}} + edit + + +
From f3009dc23b65b3d4302db8b107326f092c16baaa Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 16:06:40 +0200 Subject: [PATCH 16/41] fix(alerting): broken link --- public/app/features/alerting/partials/notifications_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/alerting/partials/notifications_list.html b/public/app/features/alerting/partials/notifications_list.html index be0a44fa0e0..10f45571c31 100644 --- a/public/app/features/alerting/partials/notifications_list.html +++ b/public/app/features/alerting/partials/notifications_list.html @@ -18,7 +18,7 @@ - + {{notification.name}} From 6705efef6fed2b0795af80502e43648e47d2357e Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 16:18:40 +0200 Subject: [PATCH 17/41] feat(alerting): make some settings properties required --- pkg/services/alerting/notifier.go | 33 ++++++-- pkg/services/alerting/notifier_test.go | 109 ++++++++++++++++--------- 2 files changed, 97 insertions(+), 45 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 6e31fed7db3..661f761b6ff 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1,6 +1,8 @@ package alerting import ( + "fmt" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" @@ -108,27 +110,44 @@ func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []* } func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, error) { + notifier, err := createNotifier(model.Type, model.Settings) + + if err != nil { + return nil, err + } + return &Notification{ Name: model.Name, Type: model.Type, - Notifierr: createNotifier(model.Type, model.Settings), + Notifierr: notifier, SendCritical: !model.Settings.Get("ignoreCrit").MustBool(), SendWarning: !model.Settings.Get("ignoreWarn").MustBool(), }, nil } -var createNotifier = func(notificationType string, settings *simplejson.Json) NotificationDispatcher { +var createNotifier = func(notificationType string, settings *simplejson.Json) (NotificationDispatcher, error) { if notificationType == "email" { - return &EmailNotifier{ - To: settings.Get("to").MustString(), - log: log.New("alerting.notification.email"), + to := settings.Get("to").MustString() + + if to == "" { + return nil, fmt.Errorf("Could not find to propertie in settings") } + + return &EmailNotifier{ + To: to, + log: log.New("alerting.notification.email"), + }, nil + } + + url := settings.Get("url").MustString() + if url == "" { + return nil, fmt.Errorf("Could not find url propertie in settings") } return &WebhookNotifier{ - Url: settings.Get("url").MustString(), + Url: url, User: settings.Get("user").MustString(), Password: settings.Get("password").MustString(), log: log.New("alerting.notification.webhook"), - } + }, nil } diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index 4249238cfa0..a7d720d01d9 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -13,54 +13,87 @@ import ( func TestAlertNotificationExtraction(t *testing.T) { Convey("Parsing alert notification from settings", t, func() { - Convey("Parsing email notification from settings", func() { - json := ` - { - "to": "ops@grafana.org" - }` + Convey("Parsing email", func() { + Convey("empty settings should return error", func() { + json := `{ }` - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "ops", - Type: "email", - Settings: settingsJSON, - } + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "email", + Settings: settingsJSON, + } - not, err := NewNotificationFromDBModel(model) + _, err := NewNotificationFromDBModel(model) + So(err, ShouldNotBeNil) + }) - So(err, ShouldBeNil) - So(not.Name, ShouldEqual, "ops") - So(not.Type, ShouldEqual, "email") - So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier") + Convey("from settings", func() { + json := ` + { + "to": "ops@grafana.org" + }` - email := not.Notifierr.(*EmailNotifier) - So(email.To, ShouldEqual, "ops@grafana.org") + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "email", + Settings: settingsJSON, + } + + not, err := NewNotificationFromDBModel(model) + + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "ops") + So(not.Type, ShouldEqual, "email") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier") + + email := not.Notifierr.(*EmailNotifier) + So(email.To, ShouldEqual, "ops@grafana.org") + }) }) - Convey("Parsing webhook notification from settings", func() { - json := ` - { - "url": "http://localhost:3000", - "username": "username", - "password": "password" - }` + Convey("Parsing webhook", func() { + Convey("empty settings should return error", func() { + json := `{ }` - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "slack", - Type: "webhook", - Settings: settingsJSON, - } + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "webhook", + Settings: settingsJSON, + } - not, err := NewNotificationFromDBModel(model) + _, err := NewNotificationFromDBModel(model) + So(err, ShouldNotBeNil) + }) - So(err, ShouldBeNil) - So(not.Name, ShouldEqual, "slack") - So(not.Type, ShouldEqual, "webhook") - So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier") + Convey("from settings", func() { + json := ` + { + "url": "http://localhost:3000", + "username": "username", + "password": "password" + }` - webhook := not.Notifierr.(*WebhookNotifier) - So(webhook.Url, ShouldEqual, "http://localhost:3000") + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "slack", + Type: "webhook", + Settings: settingsJSON, + } + + not, err := NewNotificationFromDBModel(model) + + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "slack") + So(not.Type, ShouldEqual, "webhook") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier") + + webhook := not.Notifierr.(*WebhookNotifier) + So(webhook.Url, ShouldEqual, "http://localhost:3000") + }) }) + }) } From 3c4d2b8ca1b0d7ec24c3c5ed9fc0518a039b3dec Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 16:25:23 +0200 Subject: [PATCH 18/41] feat(alerting): add basic body for webhooks --- pkg/services/alerting/notifier.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 661f761b6ff..13588373bf3 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -73,11 +73,19 @@ type WebhookNotifier struct { func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) { this.log.Info("Sending webhook") + + bodyJSON := simplejson.New() + bodyJSON.Set("name", alertResult.AlertJob.Rule.Name) + bodyJSON.Set("state", alertResult.State) + bodyJSON.Set("trigged", alertResult.TriggeredAlerts) + + body, _ := bodyJSON.MarshalJSON() + cmd := &m.SendWebhook{ Url: this.Url, User: this.User, Password: this.Password, - Body: alertResult.Description, + Body: string(body), } bus.Dispatch(cmd) From b035074613b0faf3bcdf11266763d3536011be82 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 16 Jun 2016 17:02:45 +0200 Subject: [PATCH 19/41] tech(alerting): disable update state api --- pkg/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 708fcd0ceeb..084b2f66fe5 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -247,7 +247,7 @@ func Register(r *macaron.Macaron) { r.Group("/alerts", func() { r.Group("/rules", func() { r.Get("/:alertId/states", wrap(GetAlertStates)) - r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState)) + //r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState)) r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert)) //r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates r.Get("/", wrap(GetAlerts)) From 774add94c1c74ded64e30ea6b14889042ba41a3b Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 17 Jun 2016 15:24:17 +0200 Subject: [PATCH 20/41] feat(alerting): skip warn check if crit is triggered --- pkg/services/alerting/handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/alerting/handler.go b/pkg/services/alerting/handler.go index 2b088cccf8b..d541bf851f6 100644 --- a/pkg/services/alerting/handler.go +++ b/pkg/services/alerting/handler.go @@ -109,6 +109,7 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) ActualValue: transformedValue, Name: serie.Name, }) + continue } warnResult := evalCondition(rule.Warning, transformedValue) From 212fd272526e0e2f99fd68e7f673117d03deac0e Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 17 Jun 2016 15:30:17 +0200 Subject: [PATCH 21/41] feat(alerting): add support for email notifications --- emails/templates/alert_notification.html | 29 +++ pkg/services/alerting/notifier.go | 31 ++- .../notifications/notifications_test.go | 85 ++++++++- public/emails/alert_notification.html | 177 ++++++++++++++++++ 4 files changed, 311 insertions(+), 11 deletions(-) create mode 100644 emails/templates/alert_notification.html create mode 100644 public/emails/alert_notification.html diff --git a/emails/templates/alert_notification.html b/emails/templates/alert_notification.html new file mode 100644 index 00000000000..763d05b6733 --- /dev/null +++ b/emails/templates/alert_notification.html @@ -0,0 +1,29 @@ + + +[[Subject .Subject "Grafana Alert: [ [[.State]] ] [[.Name]]" ]] + +Alertstate: [[.State]]
+[[.AlertPageUrl]]"
+[[.DashboardLink]]"
+[[.Description]]
+ +[[if eq .State "Ok"]] + Everything is Ok +[[end]] + +[[if ne .State "Ok" ]] + + + + + + + [[ range $ta := .TriggeredAlerts]] + + + + + + [[end]] +
SerieStateActual value
[[$ta.Name]][[$ta.State]][[$ta.ActualValue]]
+[[end]] diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 13588373bf3..a7214311d97 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" + "github.com/grafana/grafana/pkg/setting" ) type NotifierImpl struct { @@ -50,18 +51,28 @@ type EmailNotifier struct { func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { this.log.Info("Sending email") - cmd := &m.SendEmailCommand{ - Data: map[string]interface{}{ - "Description": alertResult.Description, - "TriggeredAlerts": alertResult.TriggeredAlerts, - }, - To: []string{this.To}, - Info: "Alert result", - Massive: false, - Template: "", + grafanaUrl := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort) + if setting.AppSubUrl != "" { + grafanaUrl += "/" + setting.AppSubUrl } - bus.Dispatch(cmd) + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{ + "Name": "Name", + "State": alertResult.State, + "Description": alertResult.Description, + "TriggeredAlerts": alertResult.TriggeredAlerts, + "DashboardLink": grafanaUrl + "/dashboard/db/alerting", + "AlertPageUrl": grafanaUrl + "/alerting", + }, + To: []string{this.To}, + Template: "alert_notification.html", + } + + err := bus.Dispatch(cmd) + if err != nil { + this.log.Error("Could not send alert notification as email", "error", err) + } } type WebhookNotifier struct { diff --git a/pkg/services/notifications/notifications_test.go b/pkg/services/notifications/notifications_test.go index 110e24cb810..bbc7a6df587 100644 --- a/pkg/services/notifications/notifications_test.go +++ b/pkg/services/notifications/notifications_test.go @@ -9,6 +9,12 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +type testTriggeredAlert struct { + ActualValue float64 + Name string + State string +} + func TestNotifications(t *testing.T) { Convey("Given the notifications service", t, func() { @@ -34,6 +40,83 @@ func TestNotifications(t *testing.T) { So(sentMsg.Subject, ShouldEqual, "Reset your Grafana password - asd@asd.com") So(sentMsg.Body, ShouldNotContainSubstring, "Subject") }) - }) + Convey("Alert notifications", func() { + Convey("When sending reset email password", func() { + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{ + "Name": "Name", + "State": "Critical", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "AlertPageUrl": "http://localhost:3000/alerting", + "TriggeredAlerts": []testTriggeredAlert{ + {Name: "desktop", State: "Critical", ActualValue: 13}, + {Name: "mobile", State: "Warn", ActualValue: 5}, + }, + }, + To: []string{"asd@asd.com "}, + Template: "alert_notification.html", + } + + err := sendEmailCommandHandler(cmd) + So(err, ShouldBeNil) + + So(sentMsg.Body, ShouldContainSubstring, "Alertstate: Critical") + So(sentMsg.Body, ShouldContainSubstring, "http://localhost:3000/dashboard/db/alerting") + So(sentMsg.Body, ShouldContainSubstring, "Critical") + So(sentMsg.Body, ShouldContainSubstring, "Warn") + So(sentMsg.Body, ShouldContainSubstring, "mobile") + So(sentMsg.Body, ShouldContainSubstring, "desktop") + + So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Critical ] ") + }) + + Convey("given critical", func() { + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{ + "Name": "Name", + "State": "Warn", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "AlertPageUrl": "http://localhost:3000/alerting", + "TriggeredAlerts": []testTriggeredAlert{ + {Name: "desktop", State: "Critical", ActualValue: 13}, + {Name: "mobile", State: "Warn", ActualValue: 5}, + }, + }, + To: []string{"asd@asd.com "}, + Template: "alert_notification.html", + } + + err := sendEmailCommandHandler(cmd) + So(err, ShouldBeNil) + So(sentMsg.Body, ShouldContainSubstring, "Alertstate: Warn") + So(sentMsg.Body, ShouldContainSubstring, "http://localhost:3000/dashboard/db/alerting") + So(sentMsg.Body, ShouldContainSubstring, "Critical") + So(sentMsg.Body, ShouldContainSubstring, "Warn") + So(sentMsg.Body, ShouldContainSubstring, "mobile") + So(sentMsg.Body, ShouldContainSubstring, "desktop") + So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Warn ]") + }) + + Convey("given ok", func() { + cmd := &m.SendEmailCommand{ + Data: map[string]interface{}{ + "Name": "Name", + "State": "Ok", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "AlertPageUrl": "http://localhost:3000/alerting", + }, + To: []string{"asd@asd.com "}, + Template: "alert_notification.html", + } + + err := sendEmailCommandHandler(cmd) + So(err, ShouldBeNil) + So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Ok ]") + }) + }) + }) } diff --git a/public/emails/alert_notification.html b/public/emails/alert_notification.html new file mode 100644 index 00000000000..1b905b8c378 --- /dev/null +++ b/public/emails/alert_notification.html @@ -0,0 +1,177 @@ + + + + + + + + + + + + + +
+
+ + + + + +
+
+ + + + + +
+ + + + + + +
+ +
+ +
+ +
+
+ + + + + + +
+ + +{{Subject .Subject "Grafana Alert: [ {{.State}} ] {{.Name}}" }} + +Alertstate: {{.State}}
+{{.AlertPageUrl}}"
+{{.DashboardLink}}"
+{{.Description}}
+ +{{if eq .State "Ok"}} + Everything is Ok +{{end}} + +{{if ne .State "Ok" }} + + + + + + + {{ range $ta := .TriggeredAlerts}} + + + + + + {{end}} +
+{{end}} + + + + + + +
+ + + + + +
+
+

+ Sent by Grafana v{{.BuildVersion}} +

+
+
+
+ + +
+
+
+ + From ea4b14ac2264205268db50bab49155cb6f964807 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 17 Jun 2016 16:40:48 +0200 Subject: [PATCH 22/41] feat(alerting): rename critical -> crit --- pkg/services/alerting/alert_rule.go | 2 +- pkg/services/alerting/alert_rule_test.go | 7 ++++--- pkg/services/alerting/handler.go | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/services/alerting/alert_rule.go b/pkg/services/alerting/alert_rule.go index 039e4dc42f3..2fcfac64cfb 100644 --- a/pkg/services/alerting/alert_rule.go +++ b/pkg/services/alerting/alert_rule.go @@ -63,7 +63,7 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) { model.State = ruleDef.State model.Frequency = ruleDef.Frequency - critical := ruleDef.Settings.Get("critical") + critical := ruleDef.Settings.Get("crit") model.Critical = Level{ Operator: critical.Get("op").MustString(), Value: critical.Get("value").MustFloat64(), diff --git a/pkg/services/alerting/alert_rule_test.go b/pkg/services/alerting/alert_rule_test.go index 8e8bd01a34b..f02ce9e40c5 100644 --- a/pkg/services/alerting/alert_rule_test.go +++ b/pkg/services/alerting/alert_rule_test.go @@ -38,7 +38,7 @@ func TestAlertRuleModel(t *testing.T) { "description": "desc2", "handler": 0, "enabled": true, - "critical": { + "crit": { "value": 20, "op": ">" }, @@ -75,11 +75,12 @@ func TestAlertRuleModel(t *testing.T) { alertRule, err := NewAlertRuleFromDBModel(alert) So(err, ShouldBeNil) - So(alertRule.Critical.Operator, ShouldEqual, ">") - So(alertRule.Critical.Value, ShouldEqual, 20) So(alertRule.Warning.Operator, ShouldEqual, ">") So(alertRule.Warning.Value, ShouldEqual, 10) + + So(alertRule.Critical.Operator, ShouldEqual, ">") + So(alertRule.Critical.Value, ShouldEqual, 20) }) }) } diff --git a/pkg/services/alerting/handler.go b/pkg/services/alerting/handler.go index d541bf851f6..553949ae91d 100644 --- a/pkg/services/alerting/handler.go +++ b/pkg/services/alerting/handler.go @@ -102,7 +102,8 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) transformedValue, _ := rule.Transformer.Transform(serie) critResult := evalCondition(rule.Critical, transformedValue) - e.log.Debug("Alert execution Crit", "name", serie.Name, "transformedValue", transformedValue, "operator", rule.Critical.Operator, "level", rule.Critical.Value, "result", critResult) + condition2 := fmt.Sprintf("%v %s %v ", transformedValue, rule.Critical.Operator, rule.Critical.Value) + e.log.Debug("Alert execution Crit", "name", serie.Name, "condition", condition2, "result", critResult) if critResult { triggeredAlert = append(triggeredAlert, &TriggeredAlert{ State: alertstates.Critical, @@ -113,7 +114,8 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) } warnResult := evalCondition(rule.Warning, transformedValue) - e.log.Debug("Alert execution Warn", "name", serie.Name, "transformedValue", transformedValue, "operator", rule.Warning.Operator, "level", rule.Warning.Value, "result", warnResult) + condition := fmt.Sprintf("%v %s %v ", transformedValue, rule.Warning.Operator, rule.Warning.Value) + e.log.Debug("Alert execution Warn", "name", serie.Name, "condition", condition, "result", warnResult) if warnResult { triggeredAlert = append(triggeredAlert, &TriggeredAlert{ State: alertstates.Warn, From adea539b8d8bcaa1382290af442c8e2141a52357 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 17 Jun 2016 16:43:14 +0200 Subject: [PATCH 23/41] feat(alerting): add link to panel png --- emails/templates/alert_notification.html | 6 +++-- pkg/services/alerting/notifier.go | 1 + .../notifications/notifications_test.go | 23 ++++++++++--------- public/emails/alert_notification.html | 6 +++-- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/emails/templates/alert_notification.html b/emails/templates/alert_notification.html index 763d05b6733..3955c3dbf1e 100644 --- a/emails/templates/alert_notification.html +++ b/emails/templates/alert_notification.html @@ -3,14 +3,16 @@ [[Subject .Subject "Grafana Alert: [ [[.State]] ] [[.Name]]" ]] Alertstate: [[.State]]
-[[.AlertPageUrl]]"
-[[.DashboardLink]]"
+[[.AlertPageUrl]]
+[[.DashboardLink]]
[[.Description]]
[[if eq .State "Ok"]] Everything is Ok [[end]] + + [[if ne .State "Ok" ]] diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index a7214311d97..f9462038b50 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -64,6 +64,7 @@ func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { "TriggeredAlerts": alertResult.TriggeredAlerts, "DashboardLink": grafanaUrl + "/dashboard/db/alerting", "AlertPageUrl": grafanaUrl + "/alerting", + "DashboardImage": grafanaUrl + "/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500", }, To: []string{this.To}, Template: "alert_notification.html", diff --git a/pkg/services/notifications/notifications_test.go b/pkg/services/notifications/notifications_test.go index bbc7a6df587..3e855eea926 100644 --- a/pkg/services/notifications/notifications_test.go +++ b/pkg/services/notifications/notifications_test.go @@ -45,11 +45,12 @@ func TestNotifications(t *testing.T) { Convey("When sending reset email password", func() { cmd := &m.SendEmailCommand{ Data: map[string]interface{}{ - "Name": "Name", - "State": "Critical", - "Description": "Description", - "DashboardLink": "http://localhost:3000/dashboard/db/alerting", - "AlertPageUrl": "http://localhost:3000/alerting", + "Name": "Name", + "State": "Critical", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "AlertPageUrl": "http://localhost:3000/alerting", + "DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500", "TriggeredAlerts": []testTriggeredAlert{ {Name: "desktop", State: "Critical", ActualValue: 13}, {Name: "mobile", State: "Warn", ActualValue: 5}, @@ -68,18 +69,18 @@ func TestNotifications(t *testing.T) { So(sentMsg.Body, ShouldContainSubstring, "Warn") So(sentMsg.Body, ShouldContainSubstring, "mobile") So(sentMsg.Body, ShouldContainSubstring, "desktop") - So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Critical ] ") }) Convey("given critical", func() { cmd := &m.SendEmailCommand{ Data: map[string]interface{}{ - "Name": "Name", - "State": "Warn", - "Description": "Description", - "DashboardLink": "http://localhost:3000/dashboard/db/alerting", - "AlertPageUrl": "http://localhost:3000/alerting", + "Name": "Name", + "State": "Warn", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500", + "AlertPageUrl": "http://localhost:3000/alerting", "TriggeredAlerts": []testTriggeredAlert{ {Name: "desktop", State: "Critical", ActualValue: 13}, {Name: "mobile", State: "Warn", ActualValue: 5}, diff --git a/public/emails/alert_notification.html b/public/emails/alert_notification.html index 1b905b8c378..4491c71a5cc 100644 --- a/public/emails/alert_notification.html +++ b/public/emails/alert_notification.html @@ -118,8 +118,8 @@ color: #FFFFFF !important; {{Subject .Subject "Grafana Alert: [ {{.State}} ] {{.Name}}" }} Alertstate: {{.State}}
-{{.AlertPageUrl}}"
-{{.DashboardLink}}"
+{{.AlertPageUrl}}
+{{.DashboardLink}}
{{.Description}}
{{if eq .State "Ok"}} @@ -127,6 +127,8 @@ Alertstate: {{.State}}
{{end}} {{if ne .State "Ok" }} + +
From 4c4164bb405b89f9fd0c3cf757a2509f304578eb Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 07:49:08 +0200 Subject: [PATCH 24/41] test(alerting): adds test email creater --- .../send_email_integration_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/services/notifications/send_email_integration_test.go diff --git a/pkg/services/notifications/send_email_integration_test.go b/pkg/services/notifications/send_email_integration_test.go new file mode 100644 index 00000000000..74e9cb5a68d --- /dev/null +++ b/pkg/services/notifications/send_email_integration_test.go @@ -0,0 +1,55 @@ +package notifications + +import ( + "io/ioutil" + "testing" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestEmailIntegrationTest(t *testing.T) { + SkipConvey("Given the notifications service", t, func() { + bus.ClearBusHandlers() + + setting.StaticRootPath = "../../../public/" + setting.Smtp.Enabled = true + setting.Smtp.TemplatesPattern = "emails/*.html" + setting.Smtp.FromAddress = "from@address.com" + + err := Init() + So(err, ShouldBeNil) + + var sentMsg *Message + addToMailQueue = func(msg *Message) { + sentMsg = msg + ioutil.WriteFile("../../../tmp/test_email.html", []byte(msg.Body), 0777) + } + + Convey("When sending reset email password", func() { + cmd := &m.SendEmailCommand{ + + Data: map[string]interface{}{ + "Name": "Name", + "State": "Critical", + "Description": "Description", + "DashboardLink": "http://localhost:3000/dashboard/db/alerting", + "AlertPageUrl": "http://localhost:3000/alerting", + "DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=3&width=1000&height=500", + + "TriggeredAlerts": []testTriggeredAlert{ + {Name: "desktop", State: "Critical", ActualValue: 13}, + {Name: "mobile", State: "Warn", ActualValue: 5}, + }, + }, + To: []string{"asd@asd.com "}, + Template: "alert_notification.html", + } + + err := sendEmailCommandHandler(cmd) + So(err, ShouldBeNil) + }) + }) +} From 0c5da9155f78b01f2e606078d85d5aa6474facce Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 11:31:20 +0200 Subject: [PATCH 25/41] feat(alerting): only expose DTO info when requesting all notifications --- pkg/api/alerting.go | 14 +++++++++++++- pkg/api/api.go | 2 +- pkg/api/dtos/alerting.go | 10 ++++++++++ .../plugins/panel/graph/partials/tab_alerting.html | 3 +++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 869fb726410..6f59527fec9 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -166,7 +166,19 @@ func GetAlertNotifications(c *middleware.Context) Response { return ApiError(500, "Failed to get alert notifications", err) } - return Json(200, query.Result) + var result []dtos.AlertNotificationDTO + + for _, notification := range query.Result { + result = append(result, dtos.AlertNotificationDTO{ + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + Created: notification.Created, + Updated: notification.Updated, + }) + } + + return Json(200, result) } func GetAlertNotificationById(c *middleware.Context) Response { diff --git a/pkg/api/api.go b/pkg/api/api.go index f1ef2427cf2..862ef3e36a1 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -263,7 +263,7 @@ func Register(r *macaron.Macaron) { r.Delete("/:notificationId", wrap(DeleteAlertNotification)) }) - r.Get("/changes", wrap(GetAlertChanges)) + //r.Get("/changes", wrap(GetAlertChanges)) }) // error test diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 2db3878f7e9..b07ea9cd8b2 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -1,5 +1,7 @@ package dtos +import "time" + type AlertRuleDTO struct { Id int64 `json:"id"` DashboardId int64 `json:"dashboardId"` @@ -19,3 +21,11 @@ type AlertRuleDTO struct { DashbboardUri string `json:"dashboardUri"` } + +type AlertNotificationDTO struct { + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` +} diff --git a/public/app/plugins/panel/graph/partials/tab_alerting.html b/public/app/plugins/panel/graph/partials/tab_alerting.html index 3cbc6042a88..7efc1d1c6c0 100644 --- a/public/app/plugins/panel/graph/partials/tab_alerting.html +++ b/public/app/plugins/panel/graph/partials/tab_alerting.html @@ -109,8 +109,11 @@
Groups + +
From a18506e2e42607a336dbf32109c725676b6efd4e Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 11:32:55 +0200 Subject: [PATCH 26/41] feat(alerting): changing notifications should require org admin --- pkg/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 862ef3e36a1..4f0a59e3411 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -261,7 +261,7 @@ func Register(r *macaron.Macaron) { r.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) r.Get("/:notificationId", wrap(GetAlertNotificationById)) r.Delete("/:notificationId", wrap(DeleteAlertNotification)) - }) + }, reqOrgAdmin) //r.Get("/changes", wrap(GetAlertChanges)) }) From 0a85efbf18d7d163392499b4f0b0e799ef69312c Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 11:44:06 +0200 Subject: [PATCH 27/41] feat(alerting): add datasource type to settings --- pkg/services/alerting/extractor.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index 63e0b1b08bd..196af8ae171 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -23,28 +23,28 @@ func NewDashAlertExtractor(dash *m.Dashboard, orgId int64) *DashAlertExtractor { } } -func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (int64, error) { +func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (*m.DataSource, error) { if dsName == "" { query := &m.GetDataSourcesQuery{OrgId: e.OrgId} if err := bus.Dispatch(query); err != nil { - return 0, err + return nil, err } else { for _, ds := range query.Result { if ds.IsDefault { - return ds.Id, nil + return ds, nil } } } } else { query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgId} if err := bus.Dispatch(query); err != nil { - return 0, err + return nil, err } else { - return query.Result.Id, nil + return query.Result, nil } } - return 0, errors.New("Could not find datasource id for " + dsName) + return nil, errors.New("Could not find datasource id for " + dsName) } func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { @@ -94,10 +94,11 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { dsName = panel.Get("datasource").MustString() } - if datasourceId, err := e.lookupDatasourceId(dsName); err != nil { + if datasource, err := e.lookupDatasourceId(dsName); err != nil { return nil, err } else { - valueQuery.SetPath([]string{"datasourceId"}, datasourceId) + valueQuery.SetPath([]string{"datasourceId"}, datasource.Id) + valueQuery.SetPath([]string{"datasourceType"}, datasource.Type) } targetQuery := target.Get("target").MustString() From fa309ec925b00797ee72e7b1053343fd014bcde5 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 15:24:48 +0200 Subject: [PATCH 28/41] feat(alerting): add default notification group --- pkg/models/alert_notifications.go | 44 ++++++++-------- pkg/services/alerting/alert_rule.go | 2 + pkg/services/alerting/notifier.go | 13 +++-- .../send_email_integration_test.go | 2 - pkg/services/sqlstore/alert_notification.go | 50 ++++++++++++++----- .../sqlstore/alert_notification_test.go | 45 +++++++++++++---- pkg/services/sqlstore/migrations/alert_mig.go | 1 + 7 files changed, 107 insertions(+), 50 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 85fdbafe9c4..3ac23438b8e 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -7,30 +7,33 @@ import ( ) type AlertNotification struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - Name string `json:"name"` - Type string `json:"type"` - Settings *simplejson.Json `json:"settings"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + AlwaysExecute bool `json:"alwaysExecute"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - OrgID int64 `json:"-"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + AlwaysExecute bool `json:"alwaysExecute"` + OrgID int64 `json:"-"` + Settings *simplejson.Json `json:"settings"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 `json:"id" binding:"Required"` - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - OrgID int64 `json:"-"` - Settings *simplejson.Json `json:"settings" binding:"Required"` + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + AlwaysExecute bool `json:"alwaysExecute"` + OrgID int64 `json:"-"` + Settings *simplejson.Json `json:"settings" binding:"Required"` Result *AlertNotification } @@ -41,10 +44,11 @@ type DeleteAlertNotificationCommand struct { } type GetAlertNotificationQuery struct { - Name string - Id int64 - Ids []int64 - OrgID int64 + Name string + Id int64 + Ids []int64 + OrgID int64 + IncludeAlwaysExecute bool Result []*AlertNotification } diff --git a/pkg/services/alerting/alert_rule.go b/pkg/services/alerting/alert_rule.go index 2fcfac64cfb..77f21255156 100644 --- a/pkg/services/alerting/alert_rule.go +++ b/pkg/services/alerting/alert_rule.go @@ -63,6 +63,8 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) { model.State = ruleDef.State model.Frequency = ruleDef.Frequency + model.NotificationGroups = []int64{1, 2} + critical := ruleDef.Settings.Get("crit") model.Critical = Level{ Operator: critical.Get("op").MustString(), diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index f9462038b50..57da39b88e9 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -22,12 +22,12 @@ func NewNotifier() *NotifierImpl { } func (n *NotifierImpl) Notify(alertResult *AlertResult) { - notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, []int64{1, 2}) + notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) for _, notifier := range notifiers { warn := alertResult.State == alertstates.Warn && notifier.SendWarning crit := alertResult.State == alertstates.Critical && notifier.SendCritical - if warn || crit { + if (warn || crit) || alertResult.State == alertstates.Ok { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) go notifier.Notifierr.Dispatch(alertResult) } @@ -109,8 +109,9 @@ type NotificationDispatcher interface { func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification { query := &m.GetAlertNotificationQuery{ - OrgID: orgId, - Ids: notificationGroups, + OrgID: orgId, + Ids: notificationGroups, + IncludeAlwaysExecute: true, } err := bus.Dispatch(query) if err != nil { @@ -118,11 +119,13 @@ func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []* } var result []*Notification - + n.log.Info("notifiriring", "count", len(query.Result), "groups", notificationGroups) for _, notification := range query.Result { not, err := NewNotificationFromDBModel(notification) if err == nil { result = append(result, not) + } else { + n.log.Error("Failed to read notification model", "error", err) } } diff --git a/pkg/services/notifications/send_email_integration_test.go b/pkg/services/notifications/send_email_integration_test.go index 74e9cb5a68d..7795921c3b7 100644 --- a/pkg/services/notifications/send_email_integration_test.go +++ b/pkg/services/notifications/send_email_integration_test.go @@ -22,9 +22,7 @@ func TestEmailIntegrationTest(t *testing.T) { err := Init() So(err, ShouldBeNil) - var sentMsg *Message addToMailQueue = func(msg *Message) { - sentMsg = msg ioutil.WriteFile("../../../tmp/test_email.html", []byte(msg.Body), 0777) } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 4c4bdb6e4c4..e40d2ba419c 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -46,7 +46,8 @@ func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Sessio alert_notification.type, alert_notification.created, alert_notification.updated, - alert_notification.settings + alert_notification.settings, + alert_notification.always_execute FROM alert_notification `) @@ -77,18 +78,43 @@ func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Sessio sql.WriteString(`)`) } - var result []*m.AlertNotification - if err := sess.Sql(sql.String(), params...).Find(&result); err != nil { + var searches []*m.AlertNotification + if err := sess.Sql(sql.String(), params...).Find(&searches); err != nil { return err } + var result []*m.AlertNotification + var def []*m.AlertNotification + if query.IncludeAlwaysExecute { + + if err := sess.Where("org_id = ? AND always_execute = 1", query.OrgID).Find(&def); err != nil { + return err + } + + result = append(result, def...) + } + + for _, s := range searches { + canAppend := true + for _, d := range result { + if d.Id == s.Id { + canAppend = false + break + } + } + + if canAppend { + result = append(result, s) + } + } + query.Result = result return nil } func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error { return inTransaction(func(sess *xorm.Session) error { - existingQuery := &m.GetAlertNotificationQuery{OrgID: cmd.OrgID, Name: cmd.Name} + existingQuery := &m.GetAlertNotificationQuery{OrgID: cmd.OrgID, Name: cmd.Name, IncludeAlwaysExecute: false} err := getAlertNotifications(existingQuery, sess) if err != nil { @@ -100,12 +126,13 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error } alertNotification := &m.AlertNotification{ - OrgId: cmd.OrgID, - Name: cmd.Name, - Type: cmd.Type, - Created: time.Now(), - Settings: cmd.Settings, - Updated: time.Now(), + OrgId: cmd.OrgID, + Name: cmd.Name, + Type: cmd.Type, + Created: time.Now(), + Settings: cmd.Settings, + Updated: time.Now(), + AlwaysExecute: cmd.AlwaysExecute, } _, err = sess.Insert(alertNotification) @@ -114,7 +141,6 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return err } - //alertNotification.Id = int(id) cmd.Result = alertNotification return nil }) @@ -137,9 +163,9 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { alertNotification.Settings = cmd.Settings alertNotification.Updated = time.Now() alertNotification.Created = current.Created + alertNotification.AlwaysExecute = cmd.AlwaysExecute var affected int64 - //affected, err = sess.Id(alertNotification.Id).Cols("name", "type", "settings", "updated").Update(alertNotification) affected, err = sess.Id(alertNotification.Id).Update(alertNotification) if err != nil { diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 0b5b8e8cf13..ea0f487aa1e 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -28,10 +28,11 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgID: 1, - Settings: simplejson.New(), + Name: "ops", + Type: "email", + OrgID: 1, + Settings: simplejson.New(), + AlwaysExecute: true, } err = CreateAlertNotificationCommand(cmd) @@ -39,6 +40,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(cmd.Result.Id, ShouldNotEqual, 0) So(cmd.Result.OrgId, ShouldNotEqual, 0) So(cmd.Result.Type, ShouldEqual, "email") + So(cmd.Result.AlwaysExecute, ShouldEqual, true) Convey("Cannot save Alert Notification with the same name", func() { err = CreateAlertNotificationCommand(cmd) @@ -47,11 +49,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can update alert notification", func() { newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: "webhook", - OrgID: cmd.Result.OrgId, - Settings: simplejson.New(), - Id: cmd.Result.Id, + Name: "NewName", + Type: "webhook", + OrgID: cmd.Result.OrgId, + Settings: simplejson.New(), + Id: cmd.Result.Id, + AlwaysExecute: true, } err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) @@ -60,6 +63,14 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can search using an array of ids", func() { + So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{ + Name: "nagios", + Type: "webhook", + OrgID: 1, + Settings: simplejson.New(), + AlwaysExecute: true, + }), ShouldBeNil) + So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{ Name: "ops2", Type: "email", @@ -75,14 +86,26 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }), ShouldBeNil) Convey("search", func() { + existingNotification := int64(2) + missingThatSholdNotCauseerrors := int64(99) + query := &m.GetAlertNotificationQuery{ - Ids: []int64{1, 2, 3}, - OrgID: 1, + Ids: []int64{existingNotification, missingThatSholdNotCauseerrors}, + OrgID: 1, + IncludeAlwaysExecute: true, } err := AlertNotificationQuery(query) So(err, ShouldBeNil) So(len(query.Result), ShouldEqual, 2) + defaultNotifications := 0 + for _, not := range query.Result { + if not.AlwaysExecute { + defaultNotifications++ + } + } + + So(defaultNotifications, ShouldEqual, 1) }) }) }) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index c49319c1c9a..a6c5f49cda1 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -74,6 +74,7 @@ func addAlertMigrations(mg *Migrator) { {Name: "org_id", Type: DB_BigInt, Nullable: false}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "always_execute", Type: DB_Bool, Nullable: false}, {Name: "settings", Type: DB_Text, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, From f5297db8f365acf9094daad15cec3a3fd914a748 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 15:30:00 +0200 Subject: [PATCH 29/41] tech(alerting): remove console log that spams tests --- public/app/plugins/panel/graph/graph.js | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/panel/graph/graph.js b/public/app/plugins/panel/graph/graph.js index cab64b9906d..af0d77ef098 100755 --- a/public/app/plugins/panel/graph/graph.js +++ b/public/app/plugins/panel/graph/graph.js @@ -271,7 +271,6 @@ function (angular, $, moment, _, kbn, GraphTooltip, thresholds) { function callPlot(incrementRenderCounter) { try { - console.log('rendering'); $.plot(elem, sortedSeries, options); } catch (e) { console.log('flotcharts error', e); From 5d62c84a197ebac17ef4b8ba2f704ecce6722915 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 16:19:15 +0200 Subject: [PATCH 30/41] feat(alerting): adds default checkbox to ui --- pkg/services/sqlstore/alert_notification.go | 21 +++++++++++-------- .../alerting/partials/notification_edit.html | 3 +++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index e40d2ba419c..3f4b9a03406 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -155,15 +155,18 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return err } - alertNotification := &m.AlertNotification{} - alertNotification.Id = cmd.Id - alertNotification.OrgId = cmd.OrgID - alertNotification.Name = cmd.Name - alertNotification.Type = cmd.Type - alertNotification.Settings = cmd.Settings - alertNotification.Updated = time.Now() - alertNotification.Created = current.Created - alertNotification.AlwaysExecute = cmd.AlwaysExecute + alertNotification := &m.AlertNotification{ + Id: cmd.Id, + OrgId: cmd.OrgID, + Name: cmd.Name, + Type: cmd.Type, + Settings: cmd.Settings, + Updated: time.Now(), + Created: current.Created, + AlwaysExecute: cmd.AlwaysExecute, + } + + sess.UseBool("always_execute") var affected int64 affected, err = sess.Id(alertNotification.Id).Update(alertNotification) diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 3c7b80ab147..26a24bcf849 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -21,6 +21,9 @@ +
+ +
From 403fdebca3a575cd1b247825a10bac219fdfeeea Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 20 Jun 2016 17:05:16 +0200 Subject: [PATCH 31/41] docs(alerting): rewrite alert docs --- docs/sources/alerting/alerting.md | 101 +++--------------------------- 1 file changed, 9 insertions(+), 92 deletions(-) diff --git a/docs/sources/alerting/alerting.md b/docs/sources/alerting/alerting.md index 47310874432..942327c2eac 100644 --- a/docs/sources/alerting/alerting.md +++ b/docs/sources/alerting/alerting.md @@ -8,104 +8,21 @@ page_keywords: alerting, grafana, plugins, documentation > Alerting is still in very early development. Please be aware. -The roadmap for alerting is described in [issue #2209](https://github.com/grafana/grafana/issues/2209#issuecomment-210077445) and the current state can be found at this page. +The roadmap for alerting in Grafana have been changing rapidly during last 2-3 months. So make sure you follow the disucssion in the [alerting issue](https://github.com/grafana/grafana/issues/2209). ## Introduction -So far Grafana does only support saving alering rules but not execute it. This means that you have to export them from grafana using the api and import them into your monitoring tool of choice. The current defintion of an alert rule looks like this: +> Alerting is turned off by default and have to be enabled in the config file. -``` go -type AlertRule struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - DashboardId int64 `json:"dashboardId"` - PanelId int64 `json:"panelId"` - Query string `json:"query"` - QueryRefId string `json:"queryRefId"` - WarnLevel int64 `json:"warnLevel"` - CritLevel int64 `json:"critLevel"` - WarnOperator string `json:"warnOperator"` - CritOperator string `json:"critOperator"` - Interval string `json:"interval"` - Title string `json:"title"` - Description string `json:"description"` - QueryRange string `json:"queryRange"` - Aggregator string `json:"aggregator"` - State string `json:"state"` -} -``` +Grafana lets you define alert rules based on metrics queries on dashboards. Every alert is connected to a panel and when ever the query for the panel is updated the alerting rule is also updated. +So far only the graph panel supports alerting. To enable alerting for a panel go to the alerting tab and press 'Create alert' button. -Most of these properties might require some extra explaination. +## Alert status page -Query: json representation of the query used by grafana. Differes depending on datasource. -QueryRange: The time range for which the query should look back. -Aggregator: How the result should be reduced into a single value. ex avg, sum, min, max -State: Current state of the alert OK, WARN, CRITICAL, ACKNOWLEGED. +You can overview all your current alerts on the alert stats page at /alerting -You can configure these settings in the Alerting tab on graph panels in edit mode. When the dashboard is saved the alert is created or updated based on the dashboard. If you wish to delete an alert you simply set the query to '- select query -' in the alerting tab and save the dashboard. +## Alert notifications -## Api +When an alert is triggered it goes to the notification handler who takes care of sending emails or push data as webhooks. +The alert notifications can be configured on /alerting/notifications -### Alert rules - -``` url -GET /api/alerts/rules -``` - -``` http -state //array of strings *optional* -dashboardId //int *optional* -panelId //int *optional* - -Result -[]AlertRule -``` - -``` http -GET /api/alerts/rules/:alertId - -Result AlertRule -``` - -### Alert state - -``` http -GET /api/alerts/rulres/:alertId/states - -Result -[ - { - alertId: int, - newState: OK, WARN, CRITICAL, ACKNOWLEGED, - created: timestamp, - info: description of what might have caused the changed alert state - } -] -``` - -``` http -PUT /api/alerts/rulres/:alertId/state -Request -{ - alertId: alertid, - newState: OK, WARN, CRITICAL, ACKNOWLEGED, - info: description of what might have caused the changed alert state -} -``` - -### Alert rule changes -``` http -GET /api/alerts/changes -limit //array of strings *optional* -sinceId //int *optional* - -Result -[ - { - id: incrementing id, - alertId: alertId, - type: CREATED/UPDATED/DELETED, - created: timestamp, - } -] -``` From 6edae37ac96fa286c3b95ce93014290721d70bad Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 22 Jun 2016 07:23:31 +0200 Subject: [PATCH 32/41] feat(alerting): rename state response method --- pkg/services/alerting/engine.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 88d39929efe..326b82875c9 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -121,20 +121,19 @@ func (e *Engine) resultHandler() { result.State = alertstates.Critical result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error) - e.saveState(result) + e.reactToState(result) } } else { result.AlertJob.ResetRetry() - e.saveState(result) + e.reactToState(result) } } } -func (e *Engine) saveState(result *AlertResult) { +func (e *Engine) reactToState(result *AlertResult) { query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id} bus.Dispatch(query) - e.notifier.Notify(result) if query.Result.ShouldUpdateState(result.State) { cmd := &m.UpdateAlertStateCommand{ AlertId: result.AlertJob.Rule.Id, @@ -146,9 +145,7 @@ func (e *Engine) saveState(result *AlertResult) { e.log.Error("Failed to save state", "error", err) } - e.log.Debug("will notify! about", "new state", result.State) - - } else { - e.log.Debug("state remains the same!") + e.log.Debug("will notify about new state", "new state", result.State) + e.notifier.Notify(result) } } From b5a29b624670891972c129a6833cfd310e5180bb Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 22 Jun 2016 07:58:30 +0200 Subject: [PATCH 33/41] test(alerting): add tests for when to send notifcations --- pkg/services/alerting/notifier.go | 64 ++++++----- pkg/services/alerting/notifier_test.go | 144 +++++++++++++++---------- 2 files changed, 121 insertions(+), 87 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 57da39b88e9..f39f148ce8d 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -12,27 +12,33 @@ import ( ) type NotifierImpl struct { - log log.Logger + log log.Logger + getNotifications func(orgId int64, notificationGroups []int64) []*Notification } func NewNotifier() *NotifierImpl { + log := log.New("alerting.notifier") return &NotifierImpl{ - log: log.New("alerting.notifier"), + log: log, + getNotifications: buildGetNotifiers(log), } } +func (n NotifierImpl) ShouldDispath(alertResult *AlertResult, notifier *Notification) bool { + warn := alertResult.State == alertstates.Warn && notifier.SendWarning + crit := alertResult.State == alertstates.Critical && notifier.SendCritical + return (warn || crit) || alertResult.State == alertstates.Ok +} + func (n *NotifierImpl) Notify(alertResult *AlertResult) { - notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) + notifiers := n.getNotifications(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) for _, notifier := range notifiers { - warn := alertResult.State == alertstates.Warn && notifier.SendWarning - crit := alertResult.State == alertstates.Critical && notifier.SendCritical - if (warn || crit) || alertResult.State == alertstates.Ok { + if n.ShouldDispath(alertResult, notifier) { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) go notifier.Notifierr.Dispatch(alertResult) } } - } type Notification struct { @@ -107,29 +113,31 @@ type NotificationDispatcher interface { Dispatch(alertResult *AlertResult) } -func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification { - query := &m.GetAlertNotificationQuery{ - OrgID: orgId, - Ids: notificationGroups, - IncludeAlwaysExecute: true, - } - err := bus.Dispatch(query) - if err != nil { - n.log.Error("Failed to read notifications", "error", err) - } - - var result []*Notification - n.log.Info("notifiriring", "count", len(query.Result), "groups", notificationGroups) - for _, notification := range query.Result { - not, err := NewNotificationFromDBModel(notification) - if err == nil { - result = append(result, not) - } else { - n.log.Error("Failed to read notification model", "error", err) +func buildGetNotifiers(log log.Logger) func(orgId int64, notificationGroups []int64) []*Notification { + return func(orgId int64, notificationGroups []int64) []*Notification { + query := &m.GetAlertNotificationQuery{ + OrgID: orgId, + Ids: notificationGroups, + IncludeAlwaysExecute: true, + } + err := bus.Dispatch(query) + if err != nil { + log.Error("Failed to read notifications", "error", err) } - } - return result + var result []*Notification + log.Info("notifiriring", "count", len(query.Result), "groups", notificationGroups) + for _, notification := range query.Result { + not, err := NewNotificationFromDBModel(notification) + if err == nil { + result = append(result, not) + } else { + log.Error("Failed to read notification model", "error", err) + } + } + + return result + } } func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, error) { diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index a7d720d01d9..4bb24661549 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -7,93 +7,119 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting/alertstates" . "github.com/smartystreets/goconvey/convey" ) func TestAlertNotificationExtraction(t *testing.T) { + Convey("Notifier tests", t, func() { + Convey("rules for sending notifications", func() { + dummieNotifier := NotifierImpl{} - Convey("Parsing alert notification from settings", t, func() { - Convey("Parsing email", func() { - Convey("empty settings should return error", func() { - json := `{ }` + result := &AlertResult{ + State: alertstates.Critical, + } - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "ops", - Type: "email", - Settings: settingsJSON, - } + notifier := &Notification{ + Name: "Test Notifier", + Type: "TestType", + SendCritical: true, + SendWarning: true, + } - _, err := NewNotificationFromDBModel(model) - So(err, ShouldNotBeNil) + Convey("Should send notification", func() { + So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeTrue) }) - Convey("from settings", func() { - json := ` + Convey("warn:false and state:warn should not send", func() { + result.State = alertstates.Warn + notifier.SendWarning = false + So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeFalse) + }) + }) + + Convey("Parsing alert notification from settings", func() { + Convey("Parsing email", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "email", + Settings: settingsJSON, + } + + _, err := NewNotificationFromDBModel(model) + So(err, ShouldNotBeNil) + }) + + Convey("from settings", func() { + json := ` { "to": "ops@grafana.org" }` - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "ops", - Type: "email", - Settings: settingsJSON, - } + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "email", + Settings: settingsJSON, + } - not, err := NewNotificationFromDBModel(model) + not, err := NewNotificationFromDBModel(model) - So(err, ShouldBeNil) - So(not.Name, ShouldEqual, "ops") - So(not.Type, ShouldEqual, "email") - So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier") + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "ops") + So(not.Type, ShouldEqual, "email") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier") - email := not.Notifierr.(*EmailNotifier) - So(email.To, ShouldEqual, "ops@grafana.org") - }) - }) - - Convey("Parsing webhook", func() { - Convey("empty settings should return error", func() { - json := `{ }` - - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "ops", - Type: "webhook", - Settings: settingsJSON, - } - - _, err := NewNotificationFromDBModel(model) - So(err, ShouldNotBeNil) + email := not.Notifierr.(*EmailNotifier) + So(email.To, ShouldEqual, "ops@grafana.org") + }) }) - Convey("from settings", func() { - json := ` + Convey("Parsing webhook", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "webhook", + Settings: settingsJSON, + } + + _, err := NewNotificationFromDBModel(model) + So(err, ShouldNotBeNil) + }) + + Convey("from settings", func() { + json := ` { "url": "http://localhost:3000", "username": "username", "password": "password" }` - settingsJSON, _ := simplejson.NewJson([]byte(json)) - model := &m.AlertNotification{ - Name: "slack", - Type: "webhook", - Settings: settingsJSON, - } + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "slack", + Type: "webhook", + Settings: settingsJSON, + } - not, err := NewNotificationFromDBModel(model) + not, err := NewNotificationFromDBModel(model) - So(err, ShouldBeNil) - So(not.Name, ShouldEqual, "slack") - So(not.Type, ShouldEqual, "webhook") - So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier") + So(err, ShouldBeNil) + So(not.Name, ShouldEqual, "slack") + So(not.Type, ShouldEqual, "webhook") + So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier") - webhook := not.Notifierr.(*WebhookNotifier) - So(webhook.Url, ShouldEqual, "http://localhost:3000") + webhook := not.Notifierr.(*WebhookNotifier) + So(webhook.Url, ShouldEqual, "http://localhost:3000") + }) }) }) - }) } From 7952723b71cf39553a009188dd0332ef214c2cc1 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 22 Jun 2016 08:09:45 +0200 Subject: [PATCH 34/41] feat(alerting): add warn/crit filter --- pkg/services/alerting/notifier.go | 4 ++-- public/app/features/alerting/notification_edit_ctrl.ts | 7 +++++++ .../app/features/alerting/partials/notification_edit.html | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index f39f148ce8d..c763e37c133 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -151,8 +151,8 @@ func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, erro Name: model.Name, Type: model.Type, Notifierr: notifier, - SendCritical: !model.Settings.Get("ignoreCrit").MustBool(), - SendWarning: !model.Settings.Get("ignoreWarn").MustBool(), + SendCritical: model.Settings.Get("sendCrit").MustBool(), + SendWarning: model.Settings.Get("sendWarn").MustBool(), }, nil } diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 08b4c41b361..9bca3cddbec 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -13,6 +13,13 @@ export class AlertNotificationEditCtrl { constructor(private $routeParams, private backendSrv, private $scope) { if ($routeParams.notificationId) { this.loadNotification($routeParams.notificationId); + } else { + this.notification = { + settings: { + sendCrit: true, + sendWarn: true, + } + }; } } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 26a24bcf849..fea91e5045a 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -24,6 +24,12 @@
+
+ +
+
+ +
From 925806df878f9dd3cd0316dea117866d564954c9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 22 Jun 2016 13:43:11 +0200 Subject: [PATCH 35/41] tech(alerting): add recovery logging --- pkg/services/alerting/engine.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 326b82875c9..a840af70eb5 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -54,7 +54,7 @@ func (e *Engine) Stop() { func (e *Engine) alertingTicker() { defer func() { if err := recover(); err != nil { - e.log.Error("Scheduler Panic, stopping...", "error", err, "stack", log.Stack(1)) + e.log.Error("Scheduler Panic: stopping alertingTicker", "error", err, "stack", log.Stack(1)) } }() @@ -75,6 +75,12 @@ func (e *Engine) alertingTicker() { } func (e *Engine) execDispatch() { + defer func() { + if err := recover(); err != nil { + e.log.Error("Scheduler Panic: stopping executor", "error", err, "stack", log.Stack(1)) + } + }() + for job := range e.execQueue { log.Trace("Alerting: engine:execDispatch() starting job %s", job.Rule.Name) job.Running = true @@ -105,6 +111,12 @@ func (e *Engine) executeJob(job *AlertJob) { } func (e *Engine) resultHandler() { + defer func() { + if err := recover(); err != nil { + e.log.Error("Engine Panic, stopping resultHandler", "error", err, "stack", log.Stack(1)) + } + }() + for result := range e.resultQueue { e.log.Debug("Alert Rule Result", "ruleId", result.AlertJob.Rule.Id, "state", result.State, "value", result.ActualValue, "retry", result.AlertJob.RetryCount) From 488b42377bf705b2b44b2e93b4c351a577c53e55 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 11:03:27 +0200 Subject: [PATCH 36/41] feat(alerting): update state if not been updated for 15min --- pkg/api/alerting.go | 1 + pkg/models/alert_state.go | 8 ++++++++ pkg/services/alerting/engine.go | 25 +++++++++++++++++++---- pkg/services/sqlstore/alert_state.go | 30 +++++++++++++++++++--------- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 6f59527fec9..4df0b6a79e3 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -140,6 +140,7 @@ func GetAlertStates(c *middleware.Context) Response { // PUT /api/alerts/events/:id func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response { cmd.AlertId = c.ParamsInt64(":alertId") + cmd.OrgId = c.OrgId query := models.GetAlertByIdQuery{Id: cmd.AlertId} if err := bus.Dispatch(&query); err != nil { diff --git a/pkg/models/alert_state.go b/pkg/models/alert_state.go index 30d442ed625..ae29b7d8a77 100644 --- a/pkg/models/alert_state.go +++ b/pkg/models/alert_state.go @@ -28,6 +28,7 @@ func (this *UpdateAlertStateCommand) IsValidState() bool { type UpdateAlertStateCommand struct { AlertId int64 `json:"alertId" binding:"Required"` + OrgId int64 `json:"orgId" binding:"Required"` NewState string `json:"newState" binding:"Required"` Info string `json:"info"` @@ -42,3 +43,10 @@ type GetAlertsStateQuery struct { Result *[]AlertState } + +type GetLastAlertStateQuery struct { + AlertId int64 + OrgId int64 + + Result *AlertState +} diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index a840af70eb5..100ea5ae0b3 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -143,14 +143,12 @@ func (e *Engine) resultHandler() { } func (e *Engine) reactToState(result *AlertResult) { - query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id} - bus.Dispatch(query) - - if query.Result.ShouldUpdateState(result.State) { + if shouldUpdateState(result) { cmd := &m.UpdateAlertStateCommand{ AlertId: result.AlertJob.Rule.Id, NewState: result.State, Info: result.Description, + OrgId: result.AlertJob.Rule.OrgId, } if err := bus.Dispatch(cmd); err != nil { @@ -161,3 +159,22 @@ func (e *Engine) reactToState(result *AlertResult) { e.notifier.Notify(result) } } + +func shouldUpdateState(result *AlertResult) bool { + query := &m.GetLastAlertStateQuery{ + AlertId: result.AlertJob.Rule.Id, + OrgId: result.AlertJob.Rule.OrgId, + } + + if err := bus.Dispatch(query); err != nil { + log.Error2("Failed to read last alert state", "error", err) + return false + } + + now := time.Now() + noEarlierState := query.Result == nil + olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + changedState := query.Result.NewState != result.State + + return noEarlierState || changedState || olderThen15Min +} diff --git a/pkg/services/sqlstore/alert_state.go b/pkg/services/sqlstore/alert_state.go index 1c64ef976bd..d2453f0cb26 100644 --- a/pkg/services/sqlstore/alert_state.go +++ b/pkg/services/sqlstore/alert_state.go @@ -12,6 +12,23 @@ import ( func init() { bus.AddHandler("sql", SetNewAlertState) bus.AddHandler("sql", GetAlertStateLogByAlertId) + bus.AddHandler("sql", GetLastAlertStateQuery) +} + +func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error { + states := make([]m.AlertState, 0) + + if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil { + return err + } + + if len(states) == 0 { + cmd.Result = nil + return fmt.Errorf("invalid amount of alertstates. Expected 1 got %v", len(states)) + } + + cmd.Result = &states[0] + return nil } func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error { @@ -30,17 +47,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error { return fmt.Errorf("Could not find alert") } - if alert.State == cmd.NewState { - cmd.Result = &m.Alert{} - return nil - } - alert.State = cmd.NewState sess.Id(alert.Id).Update(&alert) alertState := m.AlertState{ AlertId: cmd.AlertId, - OrgId: cmd.AlertId, + OrgId: cmd.OrgId, NewState: cmd.NewState, Info: cmd.Info, Created: time.Now(), @@ -54,12 +66,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error { } func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error { - alertLogs := make([]m.AlertState, 0) + states := make([]m.AlertState, 0) - if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&alertLogs); err != nil { + if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil { return err } - cmd.Result = &alertLogs + cmd.Result = &states return nil } From 67197d54f9289f8f1da3a58d4a0fe89b322c0658 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 11:14:40 +0200 Subject: [PATCH 37/41] feat(alerting): add triggeredAlerts as json to alert_state --- pkg/models/alert_state.go | 23 +++++++++++-------- pkg/services/alerting/engine.go | 10 ++++---- pkg/services/alerting/handler.go | 5 +--- pkg/services/sqlstore/alert_state.go | 11 +++++---- pkg/services/sqlstore/migrations/alert_mig.go | 1 + 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/pkg/models/alert_state.go b/pkg/models/alert_state.go index ae29b7d8a77..5dbfb2ee7cd 100644 --- a/pkg/models/alert_state.go +++ b/pkg/models/alert_state.go @@ -3,16 +3,18 @@ package models import ( "time" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/alerting/alertstates" ) type AlertState struct { - Id int64 `json:"-"` - OrgId int64 `json:"-"` - AlertId int64 `json:"alertId"` - NewState string `json:"newState"` - Created time.Time `json:"created"` - Info string `json:"info"` + Id int64 `json:"-"` + OrgId int64 `json:"-"` + AlertId int64 `json:"alertId"` + NewState string `json:"newState"` + Created time.Time `json:"created"` + Info string `json:"info"` + TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"` } func (this *UpdateAlertStateCommand) IsValidState() bool { @@ -27,10 +29,11 @@ func (this *UpdateAlertStateCommand) IsValidState() bool { // Commands type UpdateAlertStateCommand struct { - AlertId int64 `json:"alertId" binding:"Required"` - OrgId int64 `json:"orgId" binding:"Required"` - NewState string `json:"newState" binding:"Required"` - Info string `json:"info"` + AlertId int64 `json:"alertId" binding:"Required"` + OrgId int64 `json:"orgId" binding:"Required"` + NewState string `json:"newState" binding:"Required"` + Info string `json:"info"` + TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"` Result *Alert } diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 100ea5ae0b3..937505f4c6b 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -6,6 +6,7 @@ import ( "github.com/benbjohnson/clock" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" @@ -145,10 +146,11 @@ func (e *Engine) resultHandler() { func (e *Engine) reactToState(result *AlertResult) { if shouldUpdateState(result) { cmd := &m.UpdateAlertStateCommand{ - AlertId: result.AlertJob.Rule.Id, - NewState: result.State, - Info: result.Description, - OrgId: result.AlertJob.Rule.OrgId, + AlertId: result.AlertJob.Rule.Id, + NewState: result.State, + Info: result.Description, + OrgId: result.AlertJob.Rule.OrgId, + TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts), } if err := bus.Dispatch(cmd); err != nil { diff --git a/pkg/services/alerting/handler.go b/pkg/services/alerting/handler.go index 553949ae91d..68a6f08931d 100644 --- a/pkg/services/alerting/handler.go +++ b/pkg/services/alerting/handler.go @@ -126,7 +126,6 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) } executionState := alertstates.Ok - description := "" for _, raised := range triggeredAlert { if raised.State == alertstates.Critical { executionState = alertstates.Critical @@ -135,9 +134,7 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) if executionState != alertstates.Critical && raised.State == alertstates.Warn { executionState = alertstates.Warn } - - description += fmt.Sprintf(descriptionFmt, raised.ActualValue, raised.Name) } - return &AlertResult{State: executionState, Description: description, TriggeredAlerts: triggeredAlert} + return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert} } diff --git a/pkg/services/sqlstore/alert_state.go b/pkg/services/sqlstore/alert_state.go index d2453f0cb26..f2d85cf6265 100644 --- a/pkg/services/sqlstore/alert_state.go +++ b/pkg/services/sqlstore/alert_state.go @@ -51,11 +51,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error { sess.Id(alert.Id).Update(&alert) alertState := m.AlertState{ - AlertId: cmd.AlertId, - OrgId: cmd.OrgId, - NewState: cmd.NewState, - Info: cmd.Info, - Created: time.Now(), + AlertId: cmd.AlertId, + OrgId: cmd.OrgId, + NewState: cmd.NewState, + Info: cmd.Info, + Created: time.Now(), + TriggeredAlerts: cmd.TriggeredAlerts, } sess.Insert(&alertState) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index a6c5f49cda1..cc8be46f52e 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -49,6 +49,7 @@ func addAlertMigrations(mg *Migrator) { {Name: "org_id", Type: DB_BigInt, Nullable: false}, {Name: "new_state", Type: DB_NVarchar, Length: 50, Nullable: false}, {Name: "info", Type: DB_Text, Nullable: true}, + {Name: "triggered_alerts", Type: DB_Text, Nullable: true}, {Name: "created", Type: DB_DateTime, Nullable: false}, }, } From f95be63c43f9c233ed1f7b6f0d9b714498868fa5 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 12:57:10 +0200 Subject: [PATCH 38/41] feat(alerting): move response handling to seperate file --- pkg/services/alerting/engine.go | 81 +++++--------------- pkg/services/alerting/result_handler.go | 68 ++++++++++++++++ pkg/services/alerting/result_handler_test.go | 58 ++++++++++++++ 3 files changed, 147 insertions(+), 60 deletions(-) create mode 100644 pkg/services/alerting/result_handler.go create mode 100644 pkg/services/alerting/result_handler_test.go diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 937505f4c6b..4d20b13ad73 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -5,35 +5,32 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" ) type Engine struct { - execQueue chan *AlertJob - resultQueue chan *AlertResult - clock clock.Clock - ticker *Ticker - scheduler Scheduler - handler AlertingHandler - ruleReader RuleReader - log log.Logger - notifier Notifier + execQueue chan *AlertJob + resultQueue chan *AlertResult + clock clock.Clock + ticker *Ticker + scheduler Scheduler + handler AlertingHandler + ruleReader RuleReader + log log.Logger + responseHandler ResultHandler } func NewEngine() *Engine { e := &Engine{ - ticker: NewTicker(time.Now(), time.Second*0, clock.New()), - execQueue: make(chan *AlertJob, 1000), - resultQueue: make(chan *AlertResult, 1000), - scheduler: NewScheduler(), - handler: NewHandler(), - ruleReader: NewRuleReader(), - log: log.New("alerting.engine"), - notifier: NewNotifier(), + ticker: NewTicker(time.Now(), time.Second*0, clock.New()), + execQueue: make(chan *AlertJob, 1000), + resultQueue: make(chan *AlertResult, 1000), + scheduler: NewScheduler(), + handler: NewHandler(), + ruleReader: NewRuleReader(), + log: log.New("alerting.engine"), + responseHandler: NewResultHandler(), } return e @@ -134,49 +131,13 @@ func (e *Engine) resultHandler() { result.State = alertstates.Critical result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error) - e.reactToState(result) + //e.reactToState(result) + e.responseHandler.Handle(result) } } else { result.AlertJob.ResetRetry() - e.reactToState(result) + //e.reactToState(result) + e.responseHandler.Handle(result) } } } - -func (e *Engine) reactToState(result *AlertResult) { - if shouldUpdateState(result) { - cmd := &m.UpdateAlertStateCommand{ - AlertId: result.AlertJob.Rule.Id, - NewState: result.State, - Info: result.Description, - OrgId: result.AlertJob.Rule.OrgId, - TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts), - } - - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to save state", "error", err) - } - - e.log.Debug("will notify about new state", "new state", result.State) - e.notifier.Notify(result) - } -} - -func shouldUpdateState(result *AlertResult) bool { - query := &m.GetLastAlertStateQuery{ - AlertId: result.AlertJob.Rule.Id, - OrgId: result.AlertJob.Rule.OrgId, - } - - if err := bus.Dispatch(query); err != nil { - log.Error2("Failed to read last alert state", "error", err) - return false - } - - now := time.Now() - noEarlierState := query.Result == nil - olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) - changedState := query.Result.NewState != result.State - - return noEarlierState || changedState || olderThen15Min -} diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go new file mode 100644 index 00000000000..97ee346fa87 --- /dev/null +++ b/pkg/services/alerting/result_handler.go @@ -0,0 +1,68 @@ +package alerting + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" +) + +type ResultHandler interface { + Handle(result *AlertResult) +} + +type ResultHandlerImpl struct { + notifier Notifier + log log.Logger +} + +func NewResultHandler() *ResultHandlerImpl { + return &ResultHandlerImpl{ + log: log.New("alerting.responseHandler"), + notifier: NewNotifier(), + } +} + +func (handler *ResultHandlerImpl) Handle(result *AlertResult) { + if handler.shouldUpdateState(result) { + cmd := &m.UpdateAlertStateCommand{ + AlertId: result.AlertJob.Rule.Id, + NewState: result.State, + Info: result.Description, + OrgId: result.AlertJob.Rule.OrgId, + TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts), + } + + if err := bus.Dispatch(cmd); err != nil { + handler.log.Error("Failed to save state", "error", err) + } + + handler.log.Debug("will notify about new state", "new state", result.State) + handler.notifier.Notify(result) + } +} + +func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool { + query := &m.GetLastAlertStateQuery{ + AlertId: result.AlertJob.Rule.Id, + OrgId: result.AlertJob.Rule.OrgId, + } + + if err := bus.Dispatch(query); err != nil { + log.Error2("Failed to read last alert state", "error", err) + return false + } + + now := time.Now() + + if query.Result == nil { + return true + } + + olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + changedState := query.Result.NewState != result.State + + return changedState || olderThen15Min +} diff --git a/pkg/services/alerting/result_handler_test.go b/pkg/services/alerting/result_handler_test.go new file mode 100644 index 00000000000..bd492259b84 --- /dev/null +++ b/pkg/services/alerting/result_handler_test.go @@ -0,0 +1,58 @@ +package alerting + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting/alertstates" + . "github.com/smartystreets/goconvey/convey" +) + +func TestAlertResultHandler(t *testing.T) { + Convey("Test result Handler", t, func() { + resultHandler := ResultHandlerImpl{} + mockResult := &AlertResult{ + State: alertstates.Ok, + AlertJob: &AlertJob{ + Rule: &AlertRule{ + Id: 1, + OrgId: 1, + }, + }, + } + mockAlertState := &m.AlertState{} + bus.ClearBusHandlers() + bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error { + query.Result = mockAlertState + return nil + }) + + Convey("Should update", func() { + + Convey("when no earlier alert state", func() { + mockAlertState = nil + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + + Convey("alert state have changed", func() { + mockAlertState = &m.AlertState{ + NewState: alertstates.Critical, + } + mockResult.State = alertstates.Ok + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + + Convey("last alert state was 15min ago", func() { + now := time.Now() + mockAlertState = &m.AlertState{ + NewState: alertstates.Critical, + Created: now.Add(time.Minute * -30), + } + mockResult.State = alertstates.Critical + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + }) + }) +} From 8b05af2f90cc54338a51d0914405dff3752dfe7c Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 15:52:45 +0200 Subject: [PATCH 39/41] feat(alerting): add exeuction time to alertResult --- pkg/services/alerting/engine.go | 9 +++++---- pkg/services/alerting/handler.go | 10 ++++++---- pkg/services/alerting/models.go | 3 +++ pkg/services/alerting/result_handler.go | 8 +++++--- pkg/services/alerting/result_handler_test.go | 1 + 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 4d20b13ad73..6061cf42d03 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -95,10 +95,11 @@ func (e *Engine) executeJob(job *AlertJob) { select { case <-time.After(time.Second * 5): e.resultQueue <- &AlertResult{ - State: alertstates.Pending, - Duration: float64(time.Since(now).Nanoseconds()) / float64(1000000), - Error: fmt.Errorf("Timeout"), - AlertJob: job, + State: alertstates.Pending, + Duration: float64(time.Since(now).Nanoseconds()) / float64(1000000), + Error: fmt.Errorf("Timeout"), + AlertJob: job, + ExeuctionTime: time.Now(), } e.log.Debug("Job Execution timeout", "alertRuleId", job.Rule.Id) case result := <-resultChan: diff --git a/pkg/services/alerting/handler.go b/pkg/services/alerting/handler.go index 68a6f08931d..1dfc6cd2cc4 100644 --- a/pkg/services/alerting/handler.go +++ b/pkg/services/alerting/handler.go @@ -2,6 +2,7 @@ package alerting import ( "fmt" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" @@ -28,9 +29,10 @@ func (e *HandlerImpl) Execute(job *AlertJob, resultQueue chan *AlertResult) { timeSeries, err := e.executeQuery(job) if err != nil { resultQueue <- &AlertResult{ - Error: err, - State: alertstates.Pending, - AlertJob: job, + Error: err, + State: alertstates.Pending, + AlertJob: job, + ExeuctionTime: time.Now(), } } @@ -136,5 +138,5 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) } } - return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert} + return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert, ExeuctionTime: time.Now()} } diff --git a/pkg/services/alerting/models.go b/pkg/services/alerting/models.go index 3e0aefd477f..aff42603412 100644 --- a/pkg/services/alerting/models.go +++ b/pkg/services/alerting/models.go @@ -1,5 +1,7 @@ package alerting +import "time" + type AlertJob struct { Offset int64 Delay bool @@ -28,6 +30,7 @@ type AlertResult struct { Description string Error error AlertJob *AlertJob + ExeuctionTime time.Time } type TriggeredAlert struct { diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 97ee346fa87..27f2a4024fe 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -55,13 +55,15 @@ func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool { return false } - now := time.Now() - if query.Result == nil { return true } - olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + //now := time.Now() + //olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + lastExecution := query.Result.Created + asdf := result.ExeuctionTime.Add(time.Minute * -15) + olderThen15Min := lastExecution.Before(asdf) changedState := query.Result.NewState != result.State return changedState || olderThen15Min diff --git a/pkg/services/alerting/result_handler_test.go b/pkg/services/alerting/result_handler_test.go index bd492259b84..f44049ecb6d 100644 --- a/pkg/services/alerting/result_handler_test.go +++ b/pkg/services/alerting/result_handler_test.go @@ -51,6 +51,7 @@ func TestAlertResultHandler(t *testing.T) { Created: now.Add(time.Minute * -30), } mockResult.State = alertstates.Critical + mockResult.ExeuctionTime = time.Now() So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) }) }) From 6121d15ba7b8541d53e602bf3b33d1a10145b7f0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 16:07:23 +0200 Subject: [PATCH 40/41] feat(alerting): more aggressive requirements for parsing alertrules --- pkg/services/alerting/alert_rule.go | 17 +++++++++++++++-- pkg/services/alerting/alert_rule_test.go | 2 +- pkg/services/alerting/engine.go | 2 -- pkg/services/alerting/extractor_test.go | 6 +++--- pkg/services/alerting/models.go | 1 - 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/services/alerting/alert_rule.go b/pkg/services/alerting/alert_rule.go index 77f21255156..cc5968eeb70 100644 --- a/pkg/services/alerting/alert_rule.go +++ b/pkg/services/alerting/alert_rule.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strconv" + "strings" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/alerting/transformers" @@ -63,7 +64,16 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) { model.State = ruleDef.State model.Frequency = ruleDef.Frequency - model.NotificationGroups = []int64{1, 2} + ngs := ruleDef.Settings.Get("notificationGroups").MustString() + var ids []int64 + for _, v := range strings.Split(ngs, ",") { + id, err := strconv.Atoi(v) + if err == nil { + ids = append(ids, int64(id)) + } + } + + model.NotificationGroups = ids critical := ruleDef.Settings.Get("crit") model.Critical = Level{ @@ -78,6 +88,10 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) { } model.Transform = ruleDef.Settings.Get("transform").Get("type").MustString() + if model.Transform == "" { + return nil, fmt.Errorf("missing transform") + } + model.TransformParams = *ruleDef.Settings.Get("transform") if model.Transform == "aggregation" { @@ -91,7 +105,6 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) { DatasourceId: query.Get("datasourceId").MustInt64(), From: query.Get("from").MustString(), To: query.Get("to").MustString(), - Aggregator: query.Get("agg").MustString(), } if model.Query.Query == "" { diff --git a/pkg/services/alerting/alert_rule_test.go b/pkg/services/alerting/alert_rule_test.go index f02ce9e40c5..8050dd46aa9 100644 --- a/pkg/services/alerting/alert_rule_test.go +++ b/pkg/services/alerting/alert_rule_test.go @@ -55,7 +55,7 @@ func TestAlertRuleModel(t *testing.T) { "datasourceId": 1 }, "transform": { - "method": "avg", + "type": "avg", "name": "aggregation" } } diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 6061cf42d03..4e002e9eb0d 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -132,12 +132,10 @@ func (e *Engine) resultHandler() { result.State = alertstates.Critical result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error) - //e.reactToState(result) e.responseHandler.Handle(result) } } else { result.AlertJob.ResetRetry() - //e.reactToState(result) e.responseHandler.Handle(result) } } diff --git a/pkg/services/alerting/extractor_test.go b/pkg/services/alerting/extractor_test.go index 7e032d1a3ac..88fab4bf70d 100644 --- a/pkg/services/alerting/extractor_test.go +++ b/pkg/services/alerting/extractor_test.go @@ -52,8 +52,8 @@ func TestAlertRuleExtraction(t *testing.T) { "to": "now" }, "transform": { - "method": "avg", - "type": "aggregation" + "type": "avg", + "name": "aggregation" }, "warn": { "value": 10, @@ -87,7 +87,7 @@ func TestAlertRuleExtraction(t *testing.T) { "to": "now" }, "transform": { - "method": "avg", + "type": "avg", "name": "aggregation" }, "warn": { diff --git a/pkg/services/alerting/models.go b/pkg/services/alerting/models.go index aff42603412..e3b1722a689 100644 --- a/pkg/services/alerting/models.go +++ b/pkg/services/alerting/models.go @@ -47,7 +47,6 @@ type Level struct { type AlertQuery struct { Query string DatasourceId int64 - Aggregator string From string To string } From 48e1a17ac2f210c87df4deacb3285b37ee48485b Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Jun 2016 16:30:12 +0200 Subject: [PATCH 41/41] feat(alerting): remove dummie values from email notifier --- pkg/api/alerting.go | 2 +- pkg/api/playlist_play.go | 2 +- pkg/models/dashboards.go | 2 +- pkg/services/alerting/notifier.go | 28 ++++++++++++++++++++++++++-- pkg/services/sqlstore/dashboard.go | 4 ++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 4df0b6a79e3..2bf6ed9255a 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -82,7 +82,7 @@ func GetAlerts(c *middleware.Context) Response { //TODO: should be possible to speed this up with lookup table for _, alert := range alertDTOs { - for _, dash := range *dashboardsQuery.Result { + for _, dash := range dashboardsQuery.Result { if alert.DashboardId == dash.Id { alert.DashbboardUri = "db/" + dash.Slug } diff --git a/pkg/api/playlist_play.go b/pkg/api/playlist_play.go index f3bae6cbcd3..e4feb3442fb 100644 --- a/pkg/api/playlist_play.go +++ b/pkg/api/playlist_play.go @@ -18,7 +18,7 @@ func populateDashboardsById(dashboardByIds []int64) ([]m.PlaylistDashboardDto, e return result, err } - for _, item := range *dashboardQuery.Result { + for _, item := range dashboardQuery.Result { result = append(result, m.PlaylistDashboardDto{ Id: item.Id, Slug: item.Slug, diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 610f29e70aa..90e1eabc707 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -151,7 +151,7 @@ type GetDashboardTagsQuery struct { type GetDashboardsQuery struct { DashboardIds []int64 - Result *[]Dashboard + Result []*Dashboard } type GetDashboardSlugByIdQuery struct { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index c763e37c133..433935bbb2b 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -2,6 +2,7 @@ package alerting import ( "fmt" + "strconv" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" @@ -62,15 +63,38 @@ func (this *EmailNotifier) Dispatch(alertResult *AlertResult) { grafanaUrl += "/" + setting.AppSubUrl } + query := &m.GetDashboardsQuery{ + DashboardIds: []int64{alertResult.AlertJob.Rule.DashboardId}, + } + + if err := bus.Dispatch(query); err != nil { + this.log.Error("Failed to load dashboard", "error", err) + return + } + + if len(query.Result) != 1 { + this.log.Error("Can only support one dashboard", "result", len(query.Result)) + return + } + + dashboard := query.Result[0] + + panelId := strconv.Itoa(int(alertResult.AlertJob.Rule.PanelId)) + + //TODO: get from alertrule and transforms to seconds + from := "1466169458375" + to := "1466171258375" + + renderUrl := fmt.Sprintf("%s/render/dashboard-solo/db/%s?from=%s&to=%s&panelId=%s&width=1000&height=500", grafanaUrl, dashboard.Slug, from, to, panelId) cmd := &m.SendEmailCommand{ Data: map[string]interface{}{ "Name": "Name", "State": alertResult.State, "Description": alertResult.Description, "TriggeredAlerts": alertResult.TriggeredAlerts, - "DashboardLink": grafanaUrl + "/dashboard/db/alerting", + "DashboardLink": grafanaUrl + "/dashboard/db/" + dashboard.Slug, "AlertPageUrl": grafanaUrl + "/alerting", - "DashboardImage": grafanaUrl + "/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500", + "DashboardImage": renderUrl, }, To: []string{this.To}, Template: "alert_notification.html", diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index fbf245a951b..e6c5367d591 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -249,10 +249,10 @@ func GetDashboards(query *m.GetDashboardsQuery) error { return m.ErrCommandValidationFailed } - var dashboards = make([]m.Dashboard, 0) + var dashboards = make([]*m.Dashboard, 0) err := x.In("id", query.DashboardIds).Find(&dashboards) - query.Result = &dashboards + query.Result = dashboards if err != nil { return err