public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alerting): working on state management

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-07-22 13:14:09 +02:00
parent 783d697529
commit 7eb2d2cf47
23 changed files with 380 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/alerting.go
+37 -36
View File
@@ -49,6 +49,7 @@ func GetAlerts(c *middleware.Context) Response {
Name: alert.Name, Name: alert.Name,
Description: alert.Description, Description: alert.Description,
State: alert.State, State: alert.State,
Severity: alert.Severity,
}) })
} }
@@ -92,7 +93,7 @@ func AlertTest(c *middleware.Context, dto dtos.AlertTestCommand) Response {
res := backendCmd.Result res := backendCmd.Result
dtoRes := &dtos.AlertTestResult{ dtoRes := &dtos.AlertTestResult{
Triggered: res.Triggered, Firing: res.Firing,
} }
if res.Error != nil { if res.Error != nil {
@@ -138,41 +139,41 @@ func DelAlert(c *middleware.Context) Response {
return Json(200, resp) return Json(200, resp)
} }
// GET /api/alerts/events/:id // // GET /api/alerts/events/:id
func GetAlertStates(c *middleware.Context) Response { // func GetAlertStates(c *middleware.Context) Response {
alertId := c.ParamsInt64(":alertId") // alertId := c.ParamsInt64(":alertId")
//
query := models.GetAlertsStateQuery{ // query := models.GetAlertsStateQuery{
AlertId: alertId, // AlertId: alertId,
} // }
//
if err := bus.Dispatch(&query); err != nil { // if err := bus.Dispatch(&query); err != nil {
return ApiError(500, "Failed get alert state log", err) // return ApiError(500, "Failed get alert state log", err)
} // }
//
return Json(200, query.Result) // return Json(200, query.Result)
} // }
//
// PUT /api/alerts/events/:id // // PUT /api/alerts/events/:id
func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response { // func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
cmd.AlertId = c.ParamsInt64(":alertId") // cmd.AlertId = c.ParamsInt64(":alertId")
cmd.OrgId = c.OrgId // cmd.OrgId = c.OrgId
//
query := models.GetAlertByIdQuery{Id: cmd.AlertId} // query := models.GetAlertByIdQuery{Id: cmd.AlertId}
if err := bus.Dispatch(&query); err != nil { // if err := bus.Dispatch(&query); err != nil {
return ApiError(500, "Failed to get alertstate", err) // return ApiError(500, "Failed to get alertstate", err)
} // }
//
if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId { // if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId {
return ApiError(500, "Alert not found", nil) // return ApiError(500, "Alert not found", nil)
} // }
//
if err := bus.Dispatch(&cmd); err != nil { // if err := bus.Dispatch(&cmd); err != nil {
return ApiError(500, "Failed to set new state", err) // return ApiError(500, "Failed to set new state", err)
} // }
//
return Json(200, cmd.Result) // return Json(200, cmd.Result)
} // }
func GetAlertNotifications(c *middleware.Context) Response { func GetAlertNotifications(c *middleware.Context) Response {
query := &models.GetAlertNotificationQuery{ query := &models.GetAlertNotificationQuery{
pkg/api/api.go
+1 -1
View File
@@ -247,7 +247,7 @@ func Register(r *macaron.Macaron) {
r.Group("/alerts", func() { r.Group("/alerts", func() {
r.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest)) r.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest))
r.Get("/:alertId/states", wrap(GetAlertStates)) //r.Get("/:alertId/states", wrap(GetAlertStates))
//r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState)) //r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState))
r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert)) r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert))
//r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates //r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates
pkg/api/dtos/alerting.go
+12 -19
View File
@@ -4,24 +4,17 @@ import (
"time" "time"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
) )
type AlertRuleDTO struct { type AlertRuleDTO struct {
Id int64 `json:"id"` Id int64 `json:"id"`
DashboardId int64 `json:"dashboardId"` DashboardId int64 `json:"dashboardId"`
PanelId int64 `json:"panelId"` PanelId int64 `json:"panelId"`
Query string `json:"query"` Name string `json:"name"`
QueryRefId string `json:"queryRefId"` Description string `json:"description"`
WarnLevel float64 `json:"warnLevel"` State m.AlertStateType `json:"state"`
CritLevel float64 `json:"critLevel"` Severity m.AlertSeverityType `json:"severity"`
WarnOperator string `json:"warnOperator"`
CritOperator string `json:"critOperator"`
Frequency int64 `json:"frequency"`
Name string `json:"name"`
Description string `json:"description"`
QueryRange int `json:"queryRange"`
Aggregator string `json:"aggregator"`
State string `json:"state"`
DashbboardUri string `json:"dashboardUri"` DashbboardUri string `json:"dashboardUri"`
} }
@@ -40,10 +33,10 @@ type AlertTestCommand struct {
} }
type AlertTestResult struct { type AlertTestResult struct {
Triggered bool `json:"triggerd"` Firing bool `json:"firing"`
Timing string `json:"timing"` Timing string `json:"timing"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
Logs []*AlertTestResultLog `json:"logs,omitempty"` Logs []*AlertTestResultLog `json:"logs,omitempty"`
} }
type AlertTestResultLog struct { type AlertTestResultLog struct {
pkg/models/alert.go
+33 -30
View File
@@ -6,6 +6,29 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
) )
type AlertStateType string
type AlertSeverityType string
const (
AlertStatePending AlertStateType = "pending"
AlertStateFiring AlertStateType = "firing"
AlertStateOK AlertStateType = "ok"
)
func (s AlertStateType) IsValid() bool {
return s == AlertStatePending || s == AlertStateFiring || s == AlertStateOK
}
const (
AlertSeverityCritical AlertSeverityType = "critical"
AlertSeverityWarning AlertSeverityType = "warning"
AlertSeverityInfo AlertSeverityType = "info"
)
func (s AlertSeverityType) IsValid() bool {
return s == AlertSeverityCritical || s == AlertSeverityInfo || s == AlertSeverityWarning
}
type Alert struct { type Alert struct {
Id int64 Id int64
OrgId int64 OrgId int64
@@ -13,8 +36,8 @@ type Alert struct {
PanelId int64 PanelId int64
Name string Name string
Description string Description string
Severity string Severity AlertSeverityType
State string State AlertStateType
Handler int64 Handler int64
Enabled bool Enabled bool
Frequency int64 Frequency int64
@@ -32,7 +55,7 @@ func (alert *Alert) ValidToSave() bool {
return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0 return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0
} }
func (alert *Alert) ShouldUpdateState(newState string) bool { func (alert *Alert) ShouldUpdateState(newState AlertStateType) bool {
return alert.State != newState return alert.State != newState
} }
@@ -74,25 +97,6 @@ type HeartBeatCommand struct {
Result AlertingClusterInfo Result AlertingClusterInfo
} }
type AlertChange struct {
Id int64 `json:"id"`
OrgId int64 `json:"-"`
AlertId int64 `json:"alertId"`
UpdatedBy int64 `json:"updatedBy"`
NewAlertSettings *simplejson.Json `json:"newAlertSettings"`
Type string `json:"type"`
Created time.Time `json:"created"`
}
// Commands
type CreateAlertChangeCommand struct {
OrgId int64
AlertId int64
UpdatedBy int64
NewAlertSettings *simplejson.Json
Type string
}
type SaveAlertsCommand struct { type SaveAlertsCommand struct {
DashboardId int64 DashboardId int64
UserId int64 UserId int64
@@ -101,6 +105,13 @@ type SaveAlertsCommand struct {
Alerts []*Alert Alerts []*Alert
} }
type SetAlertStateCommand struct {
AlertId int64
OrgId int64
State AlertStateType
Timestamp time.Time
}
type DeleteAlertCommand struct { type DeleteAlertCommand struct {
AlertId int64 AlertId int64
} }
@@ -124,11 +135,3 @@ type GetAlertByIdQuery struct {
Result *Alert Result *Alert
} }
type GetAlertChangesQuery struct {
OrgId int64
Limit int64
SinceId int64
Result []*AlertChange
}
pkg/models/alert_state.go
+45 -52
View File
@@ -1,54 +1,47 @@
package models package models
import ( // type AlertState struct {
"time" // Id int64 `json:"-"`
// OrgId int64 `json:"-"`
"github.com/grafana/grafana/pkg/components/simplejson" // AlertId int64 `json:"alertId"`
"github.com/grafana/grafana/pkg/services/alerting/alertstates" // State string `json:"state"`
) // Created time.Time `json:"created"`
// Info string `json:"info"`
type AlertState struct { // TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
Id int64 `json:"-"` // }
OrgId int64 `json:"-"` //
AlertId int64 `json:"alertId"` // func (this *UpdateAlertStateCommand) IsValidState() bool {
State string `json:"state"` // for _, v := range alertstates.ValidStates {
Created time.Time `json:"created"` // if this.State == v {
Info string `json:"info"` // return true
TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"` // }
} // }
// return false
func (this *UpdateAlertStateCommand) IsValidState() bool { // }
for _, v := range alertstates.ValidStates { //
if this.State == v { // // Commands
return true //
} // type UpdateAlertStateCommand struct {
} // AlertId int64 `json:"alertId" binding:"Required"`
return false // OrgId int64 `json:"orgId" binding:"Required"`
} // State string `json:"state" binding:"Required"`
// Info string `json:"info"`
// Commands //
// Result *Alert
type UpdateAlertStateCommand struct { // }
AlertId int64 `json:"alertId" binding:"Required"` //
OrgId int64 `json:"orgId" binding:"Required"` // // Queries
State string `json:"state" binding:"Required"` //
Info string `json:"info"` // type GetAlertsStateQuery struct {
// OrgId int64 `json:"orgId" binding:"Required"`
Result *Alert // AlertId int64 `json:"alertId" binding:"Required"`
} //
// Result *[]AlertState
// Queries // }
//
type GetAlertsStateQuery struct { // type GetLastAlertStateQuery struct {
OrgId int64 `json:"orgId" binding:"Required"` // AlertId int64
AlertId int64 `json:"alertId" binding:"Required"` // OrgId int64
//
Result *[]AlertState // Result *AlertState
} // }
type GetLastAlertStateQuery struct {
AlertId int64
OrgId int64
Result *AlertState
}
pkg/services/alerting/alert_rule.go
+3 -1
View File
@@ -18,7 +18,8 @@ type AlertRule struct {
Frequency int64 Frequency int64
Name string Name string
Description string Description string
Severity string State m.AlertStateType
Severity m.AlertSeverityType
Conditions []AlertCondition Conditions []AlertCondition
Notifications []int64 Notifications []int64
} }
@@ -63,6 +64,7 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
model.Description = ruleDef.Description model.Description = ruleDef.Description
model.Frequency = ruleDef.Frequency model.Frequency = ruleDef.Frequency
model.Severity = ruleDef.Severity model.Severity = ruleDef.Severity
model.State = ruleDef.State
for _, v := range ruleDef.Settings.Get("notifications").MustArray() { for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
if id, ok := v.(int64); ok { if id, ok := v.(int64); ok {
pkg/services/alerting/alertstates/states.go
-16
View File
@@ -1,16 +0,0 @@
package alertstates
var (
ValidStates = []string{
Ok,
Warn,
Critical,
Unknown,
}
Ok = "OK"
Warn = "WARN"
Critical = "CRITICAL"
Pending = "PENDING"
Unknown = "UNKNOWN"
)
pkg/services/alerting/conditions.go
+1 -1
View File
@@ -40,7 +40,7 @@ func (c *QueryCondition) Eval(context *AlertResultContext) {
Metric: series.Name, Metric: series.Name,
Value: reducedValue, Value: reducedValue,
}) })
context.Triggered = true context.Firing = true
break break
} }
} }
pkg/services/alerting/conditions_test.go
+4 -4
View File
@@ -19,20 +19,20 @@ func TestQueryCondition(t *testing.T) {
ctx.reducer = `{"type": "avg"}` ctx.reducer = `{"type": "avg"}`
ctx.evaluator = `{"type": ">", "params": [100]}` ctx.evaluator = `{"type": ">", "params": [100]}`
Convey("should trigger when avg is above 100", func() { Convey("should fire when avg is above 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})} ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})}
ctx.exec() ctx.exec()
So(ctx.result.Error, ShouldBeNil) So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeTrue) So(ctx.result.Firing, ShouldBeTrue)
}) })
Convey("Should not trigger when avg is below 100", func() { Convey("Should not fire when avg is below 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})} ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})}
ctx.exec() ctx.exec()
So(ctx.result.Error, ShouldBeNil) So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeFalse) So(ctx.result.Firing, ShouldBeFalse)
}) })
}) })
}) })
pkg/services/alerting/engine.go
+1 -1
View File
@@ -100,7 +100,7 @@ func (e *Engine) resultHandler() {
}() }()
for result := range e.resultQueue { for result := range e.resultQueue {
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "triggered", result.Triggered) e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
if result.Error != nil { if result.Error != nil {
e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry") e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry")
pkg/services/alerting/extractor.go
+6 -3
View File
@@ -2,7 +2,6 @@ package alerting
import ( import (
"errors" "errors"
"fmt"
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
@@ -90,10 +89,14 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
Handler: jsonAlert.Get("handler").MustInt64(), Handler: jsonAlert.Get("handler").MustInt64(),
Enabled: jsonAlert.Get("enabled").MustBool(), Enabled: jsonAlert.Get("enabled").MustBool(),
Description: jsonAlert.Get("description").MustString(), Description: jsonAlert.Get("description").MustString(),
Severity: jsonAlert.Get("severity").MustString(), Severity: m.AlertSeverityType(jsonAlert.Get("severity").MustString()),
Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()), Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()),
} }
if !alert.Severity.IsValid() {
return nil, AlertValidationError{Reason: "Invalid alert Severity"}
}
for _, condition := range jsonAlert.Get("conditions").MustArray() { for _, condition := range jsonAlert.Get("conditions").MustArray() {
jsonCondition := simplejson.NewFromAny(condition) jsonCondition := simplejson.NewFromAny(condition)
@@ -102,7 +105,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
panelQuery := findPanelQueryByRefId(panel, queryRefId) panelQuery := findPanelQueryByRefId(panel, queryRefId)
if panelQuery == nil { if panelQuery == nil {
return nil, fmt.Errorf("Alert referes to query %s, that could not be found", queryRefId) return nil, AlertValidationError{Reason: "Alert refes to query that cannot be found"}
} }
dsName := "" dsName := ""
pkg/services/alerting/handler.go
+2 -2
View File
@@ -33,7 +33,7 @@ func (e *HandlerImpl) Execute(context *AlertResultContext) {
context.EndTime = time.Now() context.EndTime = time.Now()
e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id) e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
case <-context.DoneChan: case <-context.DoneChan:
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "triggered", context.Triggered) e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "firing", context.Firing)
} }
} }
@@ -49,7 +49,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
} }
// break if result has not triggered yet // break if result has not triggered yet
if context.Triggered == false { if context.Firing == false {
break break
} }
} }
pkg/services/alerting/handler_test.go
+7 -7
View File
@@ -7,11 +7,11 @@ import (
) )
type conditionStub struct { type conditionStub struct {
triggered bool firing bool
} }
func (c *conditionStub) Eval(context *AlertResultContext) { func (c *conditionStub) Eval(context *AlertResultContext) {
context.Triggered = c.triggered context.Firing = c.firing
} }
func TestAlertingExecutor(t *testing.T) { func TestAlertingExecutor(t *testing.T) {
@@ -21,24 +21,24 @@ func TestAlertingExecutor(t *testing.T) {
Convey("Show return triggered with single passing condition", func() { Convey("Show return triggered with single passing condition", func() {
context := NewAlertResultContext(&AlertRule{ context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{&conditionStub{ Conditions: []AlertCondition{&conditionStub{
triggered: true, firing: true,
}}, }},
}) })
handler.eval(context) handler.eval(context)
So(context.Triggered, ShouldEqual, true) So(context.Firing, ShouldEqual, true)
}) })
Convey("Show return false with not passing condition", func() { Convey("Show return false with not passing condition", func() {
context := NewAlertResultContext(&AlertRule{ context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{ Conditions: []AlertCondition{
&conditionStub{triggered: true}, &conditionStub{firing: true},
&conditionStub{triggered: false}, &conditionStub{firing: false},
}, },
}) })
handler.eval(context) handler.eval(context)
So(context.Triggered, ShouldEqual, false) So(context.Firing, ShouldEqual, false)
}) })
// Convey("Show return critical since below 2", func() { // Convey("Show return critical since below 2", func() {
pkg/services/alerting/models.go
+1 -1
View File
@@ -28,7 +28,7 @@ func (aj *AlertJob) IncRetry() {
} }
type AlertResultContext struct { type AlertResultContext struct {
Triggered bool Firing bool
IsTestRun bool IsTestRun bool
Events []*AlertEvent Events []*AlertEvent
Logs []*AlertResultLogEntry Logs []*AlertResultLogEntry
pkg/services/alerting/result_handler.go
+14 -36
View File
@@ -1,12 +1,9 @@
package alerting package alerting
import ( import (
"time"
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
) )
type ResultHandler interface { type ResultHandler interface {
@@ -20,24 +17,27 @@ type ResultHandlerImpl struct {
func NewResultHandler() *ResultHandlerImpl { func NewResultHandler() *ResultHandlerImpl {
return &ResultHandlerImpl{ return &ResultHandlerImpl{
log: log.New("alerting.responseHandler"), log: log.New("alerting.resultHandler"),
//notifier: NewNotifier(),
} }
} }
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) { func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
newState := alertstates.Ok var newState m.AlertStateType
if result.Triggered {
newState = result.Rule.Severity if result.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
newState = m.AlertStatePending
} else if result.Firing {
newState = m.AlertStateFiring
} else {
newState = m.AlertStateOK
} }
handler.log.Info("Handle result", "newState", newState) if result.Rule.State != newState {
handler.log.Info("Handle result", "triggered", result.Triggered) handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
if handler.shouldUpdateState(result, newState) { cmd := &m.SetAlertStateCommand{
cmd := &m.UpdateAlertStateCommand{
AlertId: result.Rule.Id, AlertId: result.Rule.Id,
Info: result.Description,
OrgId: result.Rule.OrgId, OrgId: result.Rule.OrgId,
State: newState, State: newState,
} }
@@ -46,30 +46,8 @@ func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
handler.log.Error("Failed to save state", "error", err) handler.log.Error("Failed to save state", "error", err)
} }
result.Rule.State = newState
//handler.log.Debug("will notify about new state", "new state", result.State) //handler.log.Debug("will notify about new state", "new state", result.State)
//handler.notifier.Notify(result) //handler.notifier.Notify(result)
} }
} }
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
query := &m.GetLastAlertStateQuery{
AlertId: result.Rule.Id,
OrgId: result.Rule.OrgId,
}
if err := bus.Dispatch(query); err != nil {
log.Error2("Failed to read last alert state", "error", err)
return false
}
if query.Result == nil {
return true
}
lastExecution := query.Result.Created
asdf := result.StartTime.Add(time.Minute * -15)
olderThen15Min := lastExecution.Before(asdf)
changedState := query.Result.State != newState
return changedState || olderThen15Min
}
pkg/services/sqlstore/alert.go
+19 -45
View File
@@ -17,52 +17,9 @@ func init() {
bus.AddHandler("sql", GetAlertById) bus.AddHandler("sql", GetAlertById)
bus.AddHandler("sql", DeleteAlertById) bus.AddHandler("sql", DeleteAlertById)
bus.AddHandler("sql", GetAllAlertQueryHandler) bus.AddHandler("sql", GetAllAlertQueryHandler)
//bus.AddHandler("sql", HeartBeat) bus.AddHandler("sql", SetAlertState)
} }
/*
func HeartBeat(query *m.HeartBeatCommand) error {
return inTransaction(func(sess *xorm.Session) error {
now := time.Now().Sub(0, 0, 0, 5)
activeTime := time.Now().Sub(0, 0, 0, 5)
ownHeartbeats := make([]m.HeartBeat, 0)
err := x.Where("server_id = ?", query.ServerId).Find(&ownHeartbeats)
if err != nil {
return err
}
if (len(ownHeartbeats)) > 0 && ownHeartbeats[0].Updated > activeTime {
//update
x.Insert(&m.HeartBeat{ServerId: query.ServerId, Created: now, Updated: now})
} else {
thisServer := ownHeartbeats[0]
thisServer.Updated = now
x.Id(thisServer.Id).Update(&thisServer)
}
activeServers := make([]m.HeartBeat, 0)
err = x.Where("server_id = ? and updated > ", query.ServerId, now.String()).OrderBy("id").Find(&activeServers)
if err != nil {
return err
}
for i, pos := range activeServers {
if pos.ServerId == query.ServerId {
query.Result = &m.AlertingClusterInfo{
ClusterSize: len(activeServers),
UptimePosition: i,
}
return nil
}
}
return nil
})
}
*/
func GetAlertById(query *m.GetAlertByIdQuery) error { func GetAlertById(query *m.GetAlertByIdQuery) error {
alert := m.Alert{} alert := m.Alert{}
has, err := x.Id(query.Id).Get(&alert) has, err := x.Id(query.Id).Get(&alert)
@@ -203,7 +160,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
} else { } else {
alert.Updated = time.Now() alert.Updated = time.Now()
alert.Created = time.Now() alert.Created = time.Now()
alert.State = "UNKNOWN" alert.State = m.AlertStatePending
alert.CreatedBy = cmd.UserId alert.CreatedBy = cmd.UserId
alert.UpdatedBy = cmd.UserId alert.UpdatedBy = cmd.UserId
@@ -253,3 +210,20 @@ func GetAlertsByDashboardId2(dashboardId int64, sess *xorm.Session) ([]*m.Alert,
return alerts, nil return alerts, nil
} }
func SetAlertState(cmd *m.SetAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error {
alert := m.Alert{}
if has, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
return err
} else if !has {
return fmt.Errorf("Could not find alert")
}
alert.State = cmd.State
sess.Id(alert.Id).Update(&alert)
return nil
})
}
pkg/services/sqlstore/alert_state.go
+75 -75
View File
@@ -1,77 +1,77 @@
package sqlstore package sqlstore
import ( // import (
"fmt" // "fmt"
"time" // "time"
//
"github.com/go-xorm/xorm" // "github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/bus" // "github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models" // m "github.com/grafana/grafana/pkg/models"
) // )
//
func init() { // func init() {
bus.AddHandler("sql", SetNewAlertState) // bus.AddHandler("sql", SetNewAlertState)
bus.AddHandler("sql", GetAlertStateLogByAlertId) // bus.AddHandler("sql", GetAlertStateLogByAlertId)
bus.AddHandler("sql", GetLastAlertStateQuery) // bus.AddHandler("sql", GetLastAlertStateQuery)
} // }
//
func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error { // func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
states := make([]m.AlertState, 0) // states := make([]m.AlertState, 0)
//
if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil { // if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
return err // return err
} // }
//
if len(states) == 0 { // if len(states) == 0 {
cmd.Result = nil // cmd.Result = nil
return nil // return nil
} // }
//
cmd.Result = &states[0] // cmd.Result = &states[0]
return nil // return nil
} // }
//
func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error { // func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error { // return inTransaction(func(sess *xorm.Session) error {
if !cmd.IsValidState() { // if !cmd.IsValidState() {
return fmt.Errorf("new state is invalid") // return fmt.Errorf("new state is invalid")
} // }
//
alert := m.Alert{} // alert := m.Alert{}
has, err := sess.Id(cmd.AlertId).Get(&alert) // has, err := sess.Id(cmd.AlertId).Get(&alert)
if err != nil { // if err != nil {
return err // return err
} // }
//
if !has { // if !has {
return fmt.Errorf("Could not find alert") // return fmt.Errorf("Could not find alert")
} // }
//
alert.State = cmd.State // alert.State = cmd.State
sess.Id(alert.Id).Update(&alert) // sess.Id(alert.Id).Update(&alert)
//
alertState := m.AlertState{ // alertState := m.AlertState{
AlertId: cmd.AlertId, // AlertId: cmd.AlertId,
OrgId: cmd.OrgId, // OrgId: cmd.OrgId,
State: cmd.State, // State: cmd.State,
Info: cmd.Info, // Info: cmd.Info,
Created: time.Now(), // Created: time.Now(),
} // }
//
sess.Insert(&alertState) // sess.Insert(&alertState)
//
cmd.Result = &alert // cmd.Result = &alert
return nil // return nil
}) // })
} // }
//
func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error { // func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
states := make([]m.AlertState, 0) // states := make([]m.AlertState, 0)
//
if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil { // if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
return err // return err
} // }
//
cmd.Result = &states // cmd.Result = &states
return nil // return nil
} // }
pkg/services/sqlstore/alert_state_test.go
+98 -98
View File
@@ -1,100 +1,100 @@
package sqlstore package sqlstore
import ( // import (
"testing" // "testing"
//
m "github.com/grafana/grafana/pkg/models" // m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey" // . "github.com/smartystreets/goconvey/convey"
) // )
//
func TestAlertingStateAccess(t *testing.T) { // func TestAlertingStateAccess(t *testing.T) {
Convey("Test alerting state changes", t, func() { // Convey("Test alerting state changes", t, func() {
InitTestDB(t) // InitTestDB(t)
//
testDash := insertTestDashboard("dashboard with alerts", 1, "alert") // testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
//
items := []*m.Alert{ // items := []*m.Alert{
{ // {
PanelId: 1, // PanelId: 1,
DashboardId: testDash.Id, // DashboardId: testDash.Id,
OrgId: testDash.OrgId, // OrgId: testDash.OrgId,
Name: "Alerting title", // Name: "Alerting title",
Description: "Alerting description", // Description: "Alerting description",
}, // },
} // }
//
cmd := m.SaveAlertsCommand{ // cmd := m.SaveAlertsCommand{
Alerts: items, // Alerts: items,
DashboardId: testDash.Id, // DashboardId: testDash.Id,
OrgId: 1, // OrgId: 1,
UserId: 1, // UserId: 1,
} // }
//
err := SaveAlerts(&cmd) // err := SaveAlerts(&cmd)
So(err, ShouldBeNil) // So(err, ShouldBeNil)
//
Convey("Cannot insert invalid states", func() { // Convey("Cannot insert invalid states", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{ // err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1, // AlertId: 1,
NewState: "maybe ok", // NewState: "maybe ok",
Info: "Shit just hit the fan", // Info: "Shit just hit the fan",
}) // })
//
So(err, ShouldNotBeNil) // So(err, ShouldNotBeNil)
}) // })
//
Convey("Changes state to alert", func() { // Convey("Changes state to alert", func() {
//
err = SetNewAlertState(&m.UpdateAlertStateCommand{ // err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1, // AlertId: 1,
NewState: "CRITICAL", // NewState: "CRITICAL",
Info: "Shit just hit the fan", // Info: "Shit just hit the fan",
}) // })
//
Convey("can get new state for alert", func() { // Convey("can get new state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1} // query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query) // err := GetAlertById(query)
So(err, ShouldBeNil) // So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "CRITICAL") // So(query.Result.State, ShouldEqual, "CRITICAL")
}) // })
//
Convey("Changes state to ok", func() { // Convey("Changes state to ok", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{ // err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1, // AlertId: 1,
NewState: "OK", // NewState: "OK",
Info: "Shit just hit the fan", // Info: "Shit just hit the fan",
}) // })
//
Convey("get ok state for alert", func() { // Convey("get ok state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1} // query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query) // err := GetAlertById(query)
So(err, ShouldBeNil) // So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "OK") // So(query.Result.State, ShouldEqual, "OK")
}) // })
//
Convey("should have two event state logs", func() { // Convey("should have two event state logs", func() {
query := &m.GetAlertsStateQuery{ // query := &m.GetAlertsStateQuery{
AlertId: 1, // AlertId: 1,
OrgId: 1, // OrgId: 1,
} // }
//
err := GetAlertStateLogByAlertId(query) // err := GetAlertStateLogByAlertId(query)
So(err, ShouldBeNil) // So(err, ShouldBeNil)
//
So(len(*query.Result), ShouldEqual, 2) // So(len(*query.Result), ShouldEqual, 2)
}) // })
//
Convey("should not get any alerts with critical state", func() { // Convey("should not get any alerts with critical state", func() {
query := &m.GetAlertsQuery{ // query := &m.GetAlertsQuery{
OrgId: 1, // OrgId: 1,
State: []string{"Critical", "Warn"}, // State: []string{"Critical", "Warn"},
} // }
//
err := HandleAlertsQuery(query) // err := HandleAlertsQuery(query)
So(err, ShouldBeNil) // So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0) // So(len(query.Result), ShouldEqual, 0)
}) // })
}) // })
}) // })
}) // })
} // }
public/app/features/alerting/alert_def.ts
+7 -8
View File
@@ -1,16 +1,15 @@
///<reference path="../../headers/common.d.ts" /> ///<reference path="../../headers/common.d.ts" />
var alertStateToCssMap = { var alertSeverityIconMap = {
"OK": "icon-gf-online alert-icon-online", "ok": "icon-gf-online alert-icon-online",
"WARN": "icon-gf-warn alert-icon-warn", "warning": "icon-gf-warn alert-icon-warn",
"CRITICAL": "icon-gf-critical alert-icon-critical", "critical": "icon-gf-critical alert-icon-critical",
"ACKNOWLEDGED": "icon-gf-alert-disabled"
}; };
function getCssForState(alertState) { function getSeverityIconClass(alertState) {
return alertStateToCssMap[alertState]; return alertSeverityIconMap[alertState];
} }
export default { export default {
getCssForState getSeverityIconClass,
}; };
public/app/features/alerting/alerts_ctrl.ts
+4 -5
View File
@@ -28,10 +28,9 @@ export class AlertListCtrl {
updateFilter() { updateFilter() {
var stats = []; var stats = [];
this.filter.ok && stats.push('Ok'); this.filter.ok && stats.push('OK');
this.filter.warn && stats.push('Warn'); this.filter.warn && stats.push('Warn');
this.filter.critical && stats.push('critical'); this.filter.critical && stats.push('critical');
this.filter.acknowleged && stats.push('acknowleged');
this.$route.current.params.state = stats; this.$route.current.params.state = stats;
this.$route.updateParams(); this.$route.updateParams();
@@ -40,10 +39,9 @@ export class AlertListCtrl {
loadAlerts() { loadAlerts() {
var stats = []; var stats = [];
this.filter.ok && stats.push('Ok'); this.filter.ok && stats.push('OK');
this.filter.warn && stats.push('Warn'); this.filter.warn && stats.push('Warn');
this.filter.critical && stats.push('critical'); this.filter.critical && stats.push('critical');
this.filter.acknowleged && stats.push('acknowleged');
var params = { var params = {
state: stats state: stats
@@ -51,7 +49,8 @@ export class AlertListCtrl {
this.backendSrv.get('/api/alerts', params).then(result => { this.backendSrv.get('/api/alerts', params).then(result => {
this.alerts = _.map(result, alert => { this.alerts = _.map(result, alert => {
alert.iconCss = alertDef.getCssForState(alert.state); alert.severityClass = alertDef.getSeverityClass(alert.severity);
alert.stateClass = alertDef.getStateClass(alert.state);
return alert; return alert;
}); });
}); });
public/app/features/alerting/partials/alert_list.html
+7 -6
View File
@@ -7,28 +7,29 @@
</div> </div>
<div class="gf-form-inline"> <div class="gf-form-inline">
<gf-form-switch class="gf-form" label="Ok" label-class="width-5" checked="ctrl.filter.ok" on-change="ctrl.updateFilter()"></gf-form-switch> <gf-form-switch class="gf-form" label="OK" label-class="width-5" checked="ctrl.filter.ok" on-change="ctrl.updateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Warn" label-class="width-5" checked="ctrl.filter.warn" on-change="ctrl.updateFilter()"></gf-form-switch> <gf-form-switch class="gf-form" label="Warn" label-class="width-5" checked="ctrl.filter.warn" on-change="ctrl.updateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Critical" label-class="width-5" checked="ctrl.filter.critical" on-change="ctrl.updateFilter()"></gf-form-switch> <gf-form-switch class="gf-form" label="Critical" label-class="width-5" checked="ctrl.filter.critical" on-change="ctrl.updateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Acknowleged" label-class="width-7" checked="ctrl.filter.acknowleged" on-change="ctrl.updateFilter()"></gf-form-switch>
</div> </div>
<table class="grafana-options-table"> <table class="grafana-options-table">
<thead> <thead>
<th style="min-width: 200px"><strong>Name</strong></th> <th style="min-width: 200px"><strong>Name</strong></th>
<th style="width: 1%">State</th> <th style="width: 1%">State</th>
<th style="width: 1%">Severity</th>
<th style="width: 1%"></th> <th style="width: 1%"></th>
</thead> </thead>
<tr ng-repeat="alert in ctrl.alerts"> <tr ng-repeat="alert in ctrl.alerts">
<td> <td>
<a href="alerting/{{alert.id}}/states"> <a href="dashboard/{{alert.dashboardUri}}?panelId={{alert.panelId}}&fullscreen&edit&editorTab=Alerting">
{{alert.name}} {{alert.name}}
</a> </a>
</td> </td>
<td class="text-center"> <td class="text-center">
<a href="alerting/{{alert.id}}/states"> {{alert.state}}
<i class="icon-gf {{alert.iconCss}}"></i> </td>
</a> <td class="text-center">
{{alert.severity}}
</td> </td>
<td class="text-center"> <td class="text-center">
<a href="dashboard/{{alert.dashboardUri}}?panelId={{alert.panelId}}&fullscreen&edit&editorTab=Alerting" class="btn btn-inverse btn-small"> <a href="dashboard/{{alert.dashboardUri}}?panelId={{alert.panelId}}&fullscreen&edit&editorTab=Alerting" class="btn btn-inverse btn-small">
public/app/plugins/panel/graph/alert_tab_ctrl.ts
+2 -2
View File
@@ -46,8 +46,8 @@ export class AlertTabCtrl {
{text: '<', value: '<'}, {text: '<', value: '<'},
]; ];
severityLevels = [ severityLevels = [
{text: 'Critical', value: 'CRITICAL'}, {text: 'Critical', value: 'critical'},
{text: 'Warning', value: 'WARN'}, {text: 'Warning', value: 'warning'},
]; ];
/** @ngInject */ /** @ngInject */
public/sass/components/_tags.scss
+1 -1
View File
@@ -34,7 +34,7 @@
} }
.label-tag:hover { .label-tag:hover {
opacity: 0.85; opacity: 0.85;
background-color: darken($purple, 10%); background-color: darken($purple, 10%);
} }