diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index d30f2697f3f..64dd619a4eb 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -21,6 +21,9 @@ type AlertRule struct { ExecutionError string `json:"executionError"` Url string `json:"url"` CanEdit bool `json:"canEdit"` + NotifyOnce bool `json:"notifyOnce"` + NotifyEval uint64 `json:"notifyEval"` + NotifyFreq uint64 `json:"notifyFrequency"` } type AlertNotification struct { diff --git a/pkg/models/alert.go b/pkg/models/alert.go index fba2aa63df9..56ceeb2cbf7 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -72,6 +72,9 @@ type Alert struct { Silenced bool ExecutionError string Frequency int64 + NotifyOnce bool + NotifyFreq uint64 + NotifyEval uint64 EvalData *simplejson.Json NewStateDate time.Time @@ -95,6 +98,8 @@ func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name result = result || this.Message != other.Message + result = result || this.NotifyOnce != other.NotifyOnce + result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq) if this.Settings != nil && other.Settings != nil { json1, err1 := this.Settings.Encode() @@ -159,6 +164,10 @@ type SetAlertStateCommand struct { Timestamp time.Time } +type IncAlertEvalCommand struct { + AlertId int64 +} + //Queries type GetAlertsQuery struct { OrgId int64 diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index e1c1bfacb2e..f820e546a93 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -122,6 +122,8 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, Handler: jsonAlert.Get("handler").MustInt64(), Message: jsonAlert.Get("message").MustString(), Frequency: frequency, + NotifyOnce: jsonAlert.Get("notifyOnce").MustBool(), + NotifyFreq: jsonAlert.Get("notifyFrequency").MustUint64(), } for _, condition := range jsonAlert.Get("conditions").MustArray() { diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 51676efdfd5..498bae3a6e6 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -32,7 +32,10 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model func defaultShouldNotify(context *alerting.EvalContext) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State { + if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce { + return false + } + if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 { return false } // Do not notify when we become OK for the first time. diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index c57b28c7c3e..56d299001f0 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,6 +88,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } + bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id}) handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 018d138dbe4..0003fe791e3 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,6 +23,9 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 + NotifyOnce bool + NotifyFreq uint64 + NotifyEval uint64 } type ValidationError struct { @@ -97,6 +100,9 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.Name = ruleDef.Name model.Message = ruleDef.Message model.Frequency = ruleDef.Frequency + model.NotifyOnce = ruleDef.NotifyOnce + model.NotifyFreq = ruleDef.NotifyFreq + model.NotifyEval = ruleDef.NotifyEval model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 58ec7e2857a..9ab28be84ee 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -22,6 +22,7 @@ func init() { bus.AddHandler("sql", GetAlertStatesForDashboard) bus.AddHandler("sql", PauseAlert) bus.AddHandler("sql", PauseAllAlerts) + bus.AddHandler("sql", IncAlertEval) } func GetAlertById(query *m.GetAlertByIdQuery) error { @@ -188,7 +189,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS if alertToUpdate.ContainsUpdates(alert) { alert.Updated = timeNow() alert.State = alertToUpdate.State - sess.MustCols("message") + sess.MustCols("message", "notify_freq", "notify_once") _, err := sess.Id(alert.Id).Update(alert) if err != nil { return err @@ -343,3 +344,22 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error return err } + +func IncAlertEval(cmd *m.IncAlertEvalCommand) error { + return inTransaction(func(sess *DBSession) error { + alert := m.Alert{} + + if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil { + return err + } + + alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq + + sess.MustCols("notify_eval") + if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 2a364d5f464..3452e5710cc 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -29,6 +29,9 @@ func addAlertMigrations(mg *Migrator) { {Name: "state_changes", Type: DB_Int, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, + {Name: "notify_once", Type: DB_Bool, Nullable: false}, + {Name: "notify_freq", Type: DB_Int, Nullable: false}, + {Name: "notify_eval", Type: DB_Int, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "id"}, Type: IndexType}, diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index 79baa1e3f5a..f0d965ae81e 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -167,6 +167,9 @@ export class AlertTabCtrl { alert.noDataState = alert.noDataState || 'no_data'; alert.executionErrorState = alert.executionErrorState || 'alerting'; alert.frequency = alert.frequency || '60s'; + alert.notifyFrequency = alert.notifyFrequency || 10; + alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce; + alert.frequency = alert.frequency || '60s'; alert.handler = alert.handler || 1; alert.notifications = alert.notifications || []; diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index cb101672aa4..084aeb2036a 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -31,6 +31,9 @@ Evaluate every + {{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }} + + evaluations