public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alerting): working on state management

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-07-22 13:14:09 +02:00
parent 783d697529
commit 7eb2d2cf47
23 changed files with 380 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/alert_rule.go
+3 -1
View File
@@ -18,7 +18,8 @@ type AlertRule struct {
Frequency int64
Name string
Description string
Severity string
State m.AlertStateType
Severity m.AlertSeverityType
Conditions []AlertCondition
Notifications []int64
}
@@ -63,6 +64,7 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
model.Description = ruleDef.Description
model.Frequency = ruleDef.Frequency
model.Severity = ruleDef.Severity
model.State = ruleDef.State
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
if id, ok := v.(int64); ok {
pkg/services/alerting/alertstates/states.go
-16
View File
@@ -1,16 +0,0 @@
package alertstates
var (
ValidStates = []string{
Ok,
Warn,
Critical,
Unknown,
}
Ok = "OK"
Warn = "WARN"
Critical = "CRITICAL"
Pending = "PENDING"
Unknown = "UNKNOWN"
)
pkg/services/alerting/conditions.go
+1 -1
View File
@@ -40,7 +40,7 @@ func (c *QueryCondition) Eval(context *AlertResultContext) {
Metric: series.Name,
Value: reducedValue,
})
context.Triggered = true
context.Firing = true
break
}
}
pkg/services/alerting/conditions_test.go
+4 -4
View File
@@ -19,20 +19,20 @@ func TestQueryCondition(t *testing.T) {
ctx.reducer = `{"type": "avg"}`
ctx.evaluator = `{"type": ">", "params": [100]}`
Convey("should trigger when avg is above 100", func() {
Convey("should fire when avg is above 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeTrue)
So(ctx.result.Firing, ShouldBeTrue)
})
Convey("Should not trigger when avg is below 100", func() {
Convey("Should not fire when avg is below 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Triggered, ShouldBeFalse)
So(ctx.result.Firing, ShouldBeFalse)
})
})
})
pkg/services/alerting/engine.go
+1 -1
View File
@@ -100,7 +100,7 @@ func (e *Engine) resultHandler() {
}()
for result := range e.resultQueue {
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "triggered", result.Triggered)
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
if result.Error != nil {
e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry")
pkg/services/alerting/extractor.go
+6 -3
View File
@@ -2,7 +2,6 @@ package alerting
import (
"errors"
"fmt"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -90,10 +89,14 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
Handler: jsonAlert.Get("handler").MustInt64(),
Enabled: jsonAlert.Get("enabled").MustBool(),
Description: jsonAlert.Get("description").MustString(),
Severity: jsonAlert.Get("severity").MustString(),
Severity: m.AlertSeverityType(jsonAlert.Get("severity").MustString()),
Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()),
}
if !alert.Severity.IsValid() {
return nil, AlertValidationError{Reason: "Invalid alert Severity"}
}
for _, condition := range jsonAlert.Get("conditions").MustArray() {
jsonCondition := simplejson.NewFromAny(condition)
@@ -102,7 +105,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
panelQuery := findPanelQueryByRefId(panel, queryRefId)
if panelQuery == nil {
return nil, fmt.Errorf("Alert referes to query %s, that could not be found", queryRefId)
return nil, AlertValidationError{Reason: "Alert refes to query that cannot be found"}
}
dsName := ""
pkg/services/alerting/handler.go
+2 -2
View File
@@ -33,7 +33,7 @@ func (e *HandlerImpl) Execute(context *AlertResultContext) {
context.EndTime = time.Now()
e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
case <-context.DoneChan:
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "triggered", context.Triggered)
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "firing", context.Firing)
}
}
@@ -49,7 +49,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
}
// break if result has not triggered yet
if context.Triggered == false {
if context.Firing == false {
break
}
}
pkg/services/alerting/handler_test.go
+7 -7
View File
@@ -7,11 +7,11 @@ import (
)
type conditionStub struct {
triggered bool
firing bool
}
func (c *conditionStub) Eval(context *AlertResultContext) {
context.Triggered = c.triggered
context.Firing = c.firing
}
func TestAlertingExecutor(t *testing.T) {
@@ -21,24 +21,24 @@ func TestAlertingExecutor(t *testing.T) {
Convey("Show return triggered with single passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{&conditionStub{
triggered: true,
firing: true,
}},
})
handler.eval(context)
So(context.Triggered, ShouldEqual, true)
So(context.Firing, ShouldEqual, true)
})
Convey("Show return false with not passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{
&conditionStub{triggered: true},
&conditionStub{triggered: false},
&conditionStub{firing: true},
&conditionStub{firing: false},
},
})
handler.eval(context)
So(context.Triggered, ShouldEqual, false)
So(context.Firing, ShouldEqual, false)
})
// Convey("Show return critical since below 2", func() {
pkg/services/alerting/models.go
+1 -1
View File
@@ -28,7 +28,7 @@ func (aj *AlertJob) IncRetry() {
}
type AlertResultContext struct {
Triggered bool
Firing bool
IsTestRun bool
Events []*AlertEvent
Logs []*AlertResultLogEntry
pkg/services/alerting/result_handler.go
+14 -36
View File
@@ -1,12 +1,9 @@
package alerting
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
)
type ResultHandler interface {
@@ -20,24 +17,27 @@ type ResultHandlerImpl struct {
func NewResultHandler() *ResultHandlerImpl {
return &ResultHandlerImpl{
log: log.New("alerting.responseHandler"),
//notifier: NewNotifier(),
log: log.New("alerting.resultHandler"),
}
}
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
newState := alertstates.Ok
if result.Triggered {
newState = result.Rule.Severity
var newState m.AlertStateType
if result.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
newState = m.AlertStatePending
} else if result.Firing {
newState = m.AlertStateFiring
} else {
newState = m.AlertStateOK
}
handler.log.Info("Handle result", "newState", newState)
handler.log.Info("Handle result", "triggered", result.Triggered)
if result.Rule.State != newState {
handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
if handler.shouldUpdateState(result, newState) {
cmd := &m.UpdateAlertStateCommand{
cmd := &m.SetAlertStateCommand{
AlertId: result.Rule.Id,
Info: result.Description,
OrgId: result.Rule.OrgId,
State: newState,
}
@@ -46,30 +46,8 @@ func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
handler.log.Error("Failed to save state", "error", err)
}
result.Rule.State = newState
//handler.log.Debug("will notify about new state", "new state", result.State)
//handler.notifier.Notify(result)
}
}
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
query := &m.GetLastAlertStateQuery{
AlertId: result.Rule.Id,
OrgId: result.Rule.OrgId,
}
if err := bus.Dispatch(query); err != nil {
log.Error2("Failed to read last alert state", "error", err)
return false
}
if query.Result == nil {
return true
}
lastExecution := query.Result.Created
asdf := result.StartTime.Add(time.Minute * -15)
olderThen15Min := lastExecution.Before(asdf)
changedState := query.Result.State != newState
return changedState || olderThen15Min
}