public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alerting): working on state management

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-07-22 13:14:09 +02:00
parent 783d697529
commit 7eb2d2cf47
23 changed files with 380 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/alert_rule.go
+3 -1
View File
@@ -18,7 +18,8 @@ type AlertRule struct {
Frequency int64
Name string
Description string
Severity string
State m.AlertStateType
Severity m.AlertSeverityType
Conditions []AlertCondition
Notifications []int64
}
@@ -63,6 +64,7 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
model.Description = ruleDef.Description
model.Frequency = ruleDef.Frequency
model.Severity = ruleDef.Severity
model.State = ruleDef.State
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
if id, ok := v.(int64); ok {
pkg/services/alerting/alertstates/states.go
-16
View File
@@ -1,16 +0,0 @@
package alertstates
var (
ValidStates = []string{
Ok,
Warn,
Critical,
Unknown,
}
Ok = "OK"
Warn = "WARN"
Critical = "CRITICAL"
Pending = "PENDING"
Unknown = "UNKNOWN"
)
pkg/services/alerting/conditions.go
+1 -1
View File
@@ -40,7 +40,7 @@ func (c *QueryCondition) Eval(context *AlertResultContext) {
Metric: series.Name,
Value: reducedValue,
})
context.Triggered = true
context.Firing = true
break
}
}
pkg/services/alerting/conditions_test.go
+4 -4
View File
@@ -19,20 +19,20 @@ func TestQueryCondition(t *testing.T) {
ctx.reducer = `{"type": "avg"}`
ctx.evaluator = `{"type": ">", "params": [100]}`
Convey("should trigger when avg is above 100", func() {
Convey("should fire when avg is above 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeTrue)
So(ctx.result.Firing, ShouldBeTrue)
})
Convey("Should not trigger when avg is below 100", func() {
Convey("Should not fire when avg is below 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeFalse)
So(ctx.result.Firing, ShouldBeFalse)
})
})
})
pkg/services/alerting/engine.go
+1 -1
View File
@@ -100,7 +100,7 @@ func (e *Engine) resultHandler() {
}()
for result := range e.resultQueue {
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "triggered", result.Triggered)
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
if result.Error != nil {
e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry")
pkg/services/alerting/extractor.go
+6 -3
View File
@@ -2,7 +2,6 @@ package alerting
import (
"errors"
"fmt"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -90,10 +89,14 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
Handler: jsonAlert.Get("handler").MustInt64(),
Enabled: jsonAlert.Get("enabled").MustBool(),
Description: jsonAlert.Get("description").MustString(),
Severity: jsonAlert.Get("severity").MustString(),
Severity: m.AlertSeverityType(jsonAlert.Get("severity").MustString()),
Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()),
}
if !alert.Severity.IsValid() {
return nil, AlertValidationError{Reason: "Invalid alert Severity"}
}
for _, condition := range jsonAlert.Get("conditions").MustArray() {
jsonCondition := simplejson.NewFromAny(condition)
@@ -102,7 +105,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
panelQuery := findPanelQueryByRefId(panel, queryRefId)
if panelQuery == nil {
return nil, fmt.Errorf("Alert referes to query %s, that could not be found", queryRefId)
return nil, AlertValidationError{Reason: "Alert refes to query that cannot be found"}
}
dsName := ""
pkg/services/alerting/handler.go
+2 -2
View File
@@ -33,7 +33,7 @@ func (e *HandlerImpl) Execute(context *AlertResultContext) {
context.EndTime = time.Now()
e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
case <-context.DoneChan:
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "triggered", context.Triggered)
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "firing", context.Firing)
}
}
@@ -49,7 +49,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
}
// break if result has not triggered yet
if context.Triggered == false {
if context.Firing == false {
break
}
}
pkg/services/alerting/handler_test.go
+7 -7
View File
@@ -7,11 +7,11 @@ import (
)
type conditionStub struct {
triggered bool
firing bool
}
func (c *conditionStub) Eval(context *AlertResultContext) {
context.Triggered = c.triggered
context.Firing = c.firing
}
func TestAlertingExecutor(t *testing.T) {
@@ -21,24 +21,24 @@ func TestAlertingExecutor(t *testing.T) {
Convey("Show return triggered with single passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{&conditionStub{
triggered: true,
firing: true,
}},
})
handler.eval(context)
So(context.Triggered, ShouldEqual, true)
So(context.Firing, ShouldEqual, true)
})
Convey("Show return false with not passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{
&conditionStub{triggered: true},
&conditionStub{triggered: false},
&conditionStub{firing: true},
&conditionStub{firing: false},
},
})
handler.eval(context)
So(context.Triggered, ShouldEqual, false)
So(context.Firing, ShouldEqual, false)
})
// Convey("Show return critical since below 2", func() {
pkg/services/alerting/models.go
+1 -1
View File
@@ -28,7 +28,7 @@ func (aj *AlertJob) IncRetry() {
}
type AlertResultContext struct {
Triggered bool
Firing bool
IsTestRun bool
Events []*AlertEvent
Logs []*AlertResultLogEntry
pkg/services/alerting/result_handler.go
+14 -36
View File
@@ -1,12 +1,9 @@
package alerting
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
)
type ResultHandler interface {
@@ -20,24 +17,27 @@ type ResultHandlerImpl struct {
func NewResultHandler() *ResultHandlerImpl {
return &ResultHandlerImpl{
log: log.New("alerting.responseHandler"),
//notifier: NewNotifier(),
log: log.New("alerting.resultHandler"),
}
}
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
newState := alertstates.Ok
if result.Triggered {
newState = result.Rule.Severity
var newState m.AlertStateType
if result.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
newState = m.AlertStatePending
} else if result.Firing {
newState = m.AlertStateFiring
} else {
newState = m.AlertStateOK
}
handler.log.Info("Handle result", "newState", newState)
handler.log.Info("Handle result", "triggered", result.Triggered)
if result.Rule.State != newState {
handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
if handler.shouldUpdateState(result, newState) {
cmd := &m.UpdateAlertStateCommand{
cmd := &m.SetAlertStateCommand{
AlertId: result.Rule.Id,
Info: result.Description,
OrgId: result.Rule.OrgId,
State: newState,
}
@@ -46,30 +46,8 @@ func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
handler.log.Error("Failed to save state", "error", err)
}
result.Rule.State = newState
//handler.log.Debug("will notify about new state", "new state", result.State)
//handler.notifier.Notify(result)
}
}
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
query := &m.GetLastAlertStateQuery{
AlertId: result.Rule.Id,
OrgId: result.Rule.OrgId,
}
if err := bus.Dispatch(query); err != nil {
log.Error2("Failed to read last alert state", "error", err)
return false
}
if query.Result == nil {
return true
}
lastExecution := query.Result.Created
asdf := result.StartTime.Add(time.Minute * -15)
olderThen15Min := lastExecution.Before(asdf)
changedState := query.Result.State != newState
return changedState || olderThen15Min
}
pkg/services/sqlstore/alert.go
+19 -45
View File
@@ -17,52 +17,9 @@ func init() {
bus.AddHandler("sql", GetAlertById)
bus.AddHandler("sql", DeleteAlertById)
bus.AddHandler("sql", GetAllAlertQueryHandler)
//bus.AddHandler("sql", HeartBeat)
bus.AddHandler("sql", SetAlertState)
}
/*
func HeartBeat(query *m.HeartBeatCommand) error {
return inTransaction(func(sess *xorm.Session) error {
now := time.Now().Sub(0, 0, 0, 5)
activeTime := time.Now().Sub(0, 0, 0, 5)
ownHeartbeats := make([]m.HeartBeat, 0)
err := x.Where("server_id = ?", query.ServerId).Find(&ownHeartbeats)
if err != nil {
return err
}
if (len(ownHeartbeats)) > 0 && ownHeartbeats[0].Updated > activeTime {
//update
x.Insert(&m.HeartBeat{ServerId: query.ServerId, Created: now, Updated: now})
} else {
thisServer := ownHeartbeats[0]
thisServer.Updated = now
x.Id(thisServer.Id).Update(&thisServer)
}
activeServers := make([]m.HeartBeat, 0)
err = x.Where("server_id = ? and updated > ", query.ServerId, now.String()).OrderBy("id").Find(&activeServers)
if err != nil {
return err
}
for i, pos := range activeServers {
if pos.ServerId == query.ServerId {
query.Result = &m.AlertingClusterInfo{
ClusterSize: len(activeServers),
UptimePosition: i,
}
return nil
}
}
return nil
})
}
*/
func GetAlertById(query *m.GetAlertByIdQuery) error {
alert := m.Alert{}
has, err := x.Id(query.Id).Get(&alert)
@@ -203,7 +160,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
} else {
alert.Updated = time.Now()
alert.Created = time.Now()
alert.State = "UNKNOWN"
alert.State = m.AlertStatePending
alert.CreatedBy = cmd.UserId
alert.UpdatedBy = cmd.UserId
@@ -253,3 +210,20 @@ func GetAlertsByDashboardId2(dashboardId int64, sess *xorm.Session) ([]*m.Alert,
return alerts, nil
}
func SetAlertState(cmd *m.SetAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error {
alert := m.Alert{}
if has, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
return err
} else if !has {
return fmt.Errorf("Could not find alert")
}
alert.State = cmd.State
sess.Id(alert.Id).Update(&alert)
return nil
})
}
pkg/services/sqlstore/alert_state.go
+75 -75
View File
@@ -1,77 +1,77 @@
package sqlstore
import (
"fmt"
"time"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
)
func init() {
bus.AddHandler("sql", SetNewAlertState)
bus.AddHandler("sql", GetAlertStateLogByAlertId)
bus.AddHandler("sql", GetLastAlertStateQuery)
}
func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
states := make([]m.AlertState, 0)
if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
return err
}
if len(states) == 0 {
cmd.Result = nil
return nil
}
cmd.Result = &states[0]
return nil
}
func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error {
if !cmd.IsValidState() {
return fmt.Errorf("new state is invalid")
}
alert := m.Alert{}
has, err := sess.Id(cmd.AlertId).Get(&alert)
if err != nil {
return err
}
if !has {
return fmt.Errorf("Could not find alert")
}
alert.State = cmd.State
sess.Id(alert.Id).Update(&alert)
alertState := m.AlertState{
AlertId: cmd.AlertId,
OrgId: cmd.OrgId,
State: cmd.State,
Info: cmd.Info,
Created: time.Now(),
}
sess.Insert(&alertState)
cmd.Result = &alert
return nil
})
}
func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
states := make([]m.AlertState, 0)
if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
return err
}
cmd.Result = &states
return nil
}
// import (
// "fmt"
// "time"
//
// "github.com/go-xorm/xorm"
// "github.com/grafana/grafana/pkg/bus"
// m "github.com/grafana/grafana/pkg/models"
// )
//
// func init() {
// bus.AddHandler("sql", SetNewAlertState)
// bus.AddHandler("sql", GetAlertStateLogByAlertId)
// bus.AddHandler("sql", GetLastAlertStateQuery)
// }
//
// func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
// states := make([]m.AlertState, 0)
//
// if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
// return err
// }
//
// if len(states) == 0 {
// cmd.Result = nil
// return nil
// }
//
// cmd.Result = &states[0]
// return nil
// }
//
// func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
// return inTransaction(func(sess *xorm.Session) error {
// if !cmd.IsValidState() {
// return fmt.Errorf("new state is invalid")
// }
//
// alert := m.Alert{}
// has, err := sess.Id(cmd.AlertId).Get(&alert)
// if err != nil {
// return err
// }
//
// if !has {
// return fmt.Errorf("Could not find alert")
// }
//
// alert.State = cmd.State
// sess.Id(alert.Id).Update(&alert)
//
// alertState := m.AlertState{
// AlertId: cmd.AlertId,
// OrgId: cmd.OrgId,
// State: cmd.State,
// Info: cmd.Info,
// Created: time.Now(),
// }
//
// sess.Insert(&alertState)
//
// cmd.Result = &alert
// return nil
// })
// }
//
// func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
// states := make([]m.AlertState, 0)
//
// if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
// return err
// }
//
// cmd.Result = &states
// return nil
// }
pkg/services/sqlstore/alert_state_test.go
+98 -98
View File
@@ -1,100 +1,100 @@
package sqlstore
import (
"testing"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertingStateAccess(t *testing.T) {
Convey("Test alerting state changes", t, func() {
InitTestDB(t)
testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
items := []*m.Alert{
{
PanelId: 1,
DashboardId: testDash.Id,
OrgId: testDash.OrgId,
Name: "Alerting title",
Description: "Alerting description",
},
}
cmd := m.SaveAlertsCommand{
Alerts: items,
DashboardId: testDash.Id,
OrgId: 1,
UserId: 1,
}
err := SaveAlerts(&cmd)
So(err, ShouldBeNil)
Convey("Cannot insert invalid states", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "maybe ok",
Info: "Shit just hit the fan",
})
So(err, ShouldNotBeNil)
})
Convey("Changes state to alert", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "CRITICAL",
Info: "Shit just hit the fan",
})
Convey("can get new state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query)
So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "CRITICAL")
})
Convey("Changes state to ok", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "OK",
Info: "Shit just hit the fan",
})
Convey("get ok state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query)
So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "OK")
})
Convey("should have two event state logs", func() {
query := &m.GetAlertsStateQuery{
AlertId: 1,
OrgId: 1,
}
err := GetAlertStateLogByAlertId(query)
So(err, ShouldBeNil)
So(len(*query.Result), ShouldEqual, 2)
})
Convey("should not get any alerts with critical state", func() {
query := &m.GetAlertsQuery{
OrgId: 1,
State: []string{"Critical", "Warn"},
}
err := HandleAlertsQuery(query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
})
})
})
}
// import (
// "testing"
//
// m "github.com/grafana/grafana/pkg/models"
// . "github.com/smartystreets/goconvey/convey"
// )
//
// func TestAlertingStateAccess(t *testing.T) {
// Convey("Test alerting state changes", t, func() {
// InitTestDB(t)
//
// testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
//
// items := []*m.Alert{
// {
// PanelId: 1,
// DashboardId: testDash.Id,
// OrgId: testDash.OrgId,
// Name: "Alerting title",
// Description: "Alerting description",
// },
// }
//
// cmd := m.SaveAlertsCommand{
// Alerts: items,
// DashboardId: testDash.Id,
// OrgId: 1,
// UserId: 1,
// }
//
// err := SaveAlerts(&cmd)
// So(err, ShouldBeNil)
//
// Convey("Cannot insert invalid states", func() {
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "maybe ok",
// Info: "Shit just hit the fan",
// })
//
// So(err, ShouldNotBeNil)
// })
//
// Convey("Changes state to alert", func() {
//
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "CRITICAL",
// Info: "Shit just hit the fan",
// })
//
// Convey("can get new state for alert", func() {
// query := &m.GetAlertByIdQuery{Id: 1}
// err := GetAlertById(query)
// So(err, ShouldBeNil)
// So(query.Result.State, ShouldEqual, "CRITICAL")
// })
//
// Convey("Changes state to ok", func() {
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "OK",
// Info: "Shit just hit the fan",
// })
//
// Convey("get ok state for alert", func() {
// query := &m.GetAlertByIdQuery{Id: 1}
// err := GetAlertById(query)
// So(err, ShouldBeNil)
// So(query.Result.State, ShouldEqual, "OK")
// })
//
// Convey("should have two event state logs", func() {
// query := &m.GetAlertsStateQuery{
// AlertId: 1,
// OrgId: 1,
// }
//
// err := GetAlertStateLogByAlertId(query)
// So(err, ShouldBeNil)
//
// So(len(*query.Result), ShouldEqual, 2)
// })
//
// Convey("should not get any alerts with critical state", func() {
// query := &m.GetAlertsQuery{
// OrgId: 1,
// State: []string{"Critical", "Warn"},
// }
//
// err := HandleAlertsQuery(query)
// So(err, ShouldBeNil)
// So(len(query.Result), ShouldEqual, 0)
// })
// })
// })
// })
// }