diff --git a/pkg/log/log.go b/pkg/log/log.go index 58f3cb89cab..a515f5c1062 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -12,6 +12,7 @@ import ( "gopkg.in/ini.v1" + "github.com/go-stack/stack" "github.com/inconshreveable/log15" "github.com/inconshreveable/log15/term" ) @@ -228,3 +229,9 @@ func LogFilterHandler(maxLevel log15.Lvl, filters map[string]log15.Lvl, h log15. return r.Lvl <= maxLevel }, h) } + +func Stack(skip int) string { + call := stack.Caller(skip) + s := stack.Trace().TrimBelow(call).TrimRuntime() + return s.String() +} diff --git a/pkg/services/alerting/commands.go b/pkg/services/alerting/commands.go index 0f2cabd2bd3..4dba9c65685 100644 --- a/pkg/services/alerting/commands.go +++ b/pkg/services/alerting/commands.go @@ -38,6 +38,10 @@ func updateDashboardAlerts(cmd *UpdateDashboardAlertsCommand) error { return nil } +func getTimeDurationStringToSeconds(str string) int64 { + return 60 +} + func ConvetAlertModelToAlertRule(ruleDef *m.Alert) (*AlertRule, error) { model := &AlertRule{} model.Id = ruleDef.Id @@ -58,7 +62,7 @@ func ConvetAlertModelToAlertRule(ruleDef *m.Alert) (*AlertRule, error) { Level: warning.Get("level").MustFloat64(), } - model.Frequency = ruleDef.Expression.Get("frequency").MustInt64() + model.Frequency = getTimeDurationStringToSeconds(ruleDef.Expression.Get("frequency").MustString()) model.Transform = ruleDef.Expression.Get("transform").Get("type").MustString() model.TransformParams = *ruleDef.Expression.Get("transform") diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 11eee8d8302..862e6993e44 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -50,6 +50,12 @@ func (e *Engine) Stop() { } func (e *Engine) alertingTicker() { + defer func() { + if err := recover(); err != nil { + e.log.Error("Scheduler Panic, stopping...", "error", err, "stack", log.Stack(1)) + } + }() + tickIndex := 0 for { diff --git a/pkg/services/alerting/rule_reader.go b/pkg/services/alerting/reader.go similarity index 100% rename from pkg/services/alerting/rule_reader.go rename to pkg/services/alerting/reader.go diff --git a/pkg/services/alerting/scheduler.go b/pkg/services/alerting/scheduler.go index 5b376e8c9d8..c8c99e19032 100644 --- a/pkg/services/alerting/scheduler.go +++ b/pkg/services/alerting/scheduler.go @@ -18,12 +18,12 @@ func NewScheduler() Scheduler { } } -func (s *SchedulerImpl) Update(rules []*AlertRule) { - s.log.Debug("Scheduler: Update") +func (s *SchedulerImpl) Update(alerts []*AlertRule) { + s.log.Debug("Scheduling update", "alerts.count", len(alerts)) jobs := make(map[int64]*AlertJob, 0) - for i, rule := range rules { + for i, rule := range alerts { var job *AlertJob if s.jobs[rule.Id] != nil { job = s.jobs[rule.Id] @@ -40,7 +40,6 @@ func (s *SchedulerImpl) Update(rules []*AlertRule) { jobs[rule.Id] = job } - s.log.Debug("Scheduler: Selected new jobs", "job count", len(jobs)) s.jobs = jobs }