public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'alert_conditions' into alerting

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-07-27 09:52:42 +02:00
parent c675ffc682 0b0f6b0848
commit cde1bbff78
64 changed files with 2354 additions and 1996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/alerting.go
+83 -47
View File
@@ -1,10 +1,13 @@
package api
import (
"fmt"
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
func ValidateOrgAlert(c *middleware.Context) {
@@ -36,16 +39,17 @@ func GetAlerts(c *middleware.Context) Response {
}
dashboardIds := make([]int64, 0)
alertDTOs := make([]*dtos.AlertRuleDTO, 0)
alertDTOs := make([]*dtos.AlertRule, 0)
for _, alert := range query.Result {
dashboardIds = append(dashboardIds, alert.DashboardId)
alertDTOs = append(alertDTOs, &dtos.AlertRuleDTO{
alertDTOs = append(alertDTOs, &dtos.AlertRule{
Id: alert.Id,
DashboardId: alert.DashboardId,
PanelId: alert.PanelId,
Name: alert.Name,
Description: alert.Description,
State: alert.State,
Severity: alert.Severity,
})
}
@@ -71,6 +75,40 @@ func GetAlerts(c *middleware.Context) Response {
return Json(200, alertDTOs)
}
// POST /api/alerts/test
func AlertTest(c *middleware.Context, dto dtos.AlertTestCommand) Response {
backendCmd := alerting.AlertTestCommand{
OrgId: c.OrgId,
Dashboard: dto.Dashboard,
PanelId: dto.PanelId,
}
if err := bus.Dispatch(&backendCmd); err != nil {
if validationErr, ok := err.(alerting.AlertValidationError); ok {
return ApiError(422, validationErr.Error(), nil)
}
return ApiError(500, "Failed to test rule", err)
}
res := backendCmd.Result
dtoRes := &dtos.AlertTestResult{
Firing: res.Firing,
}
if res.Error != nil {
dtoRes.Error = res.Error.Error()
}
for _, log := range res.Logs {
dtoRes.Logs = append(dtoRes.Logs, &dtos.AlertTestResultLog{Message: log.Message, Data: log.Data})
}
dtoRes.TimeMs = fmt.Sprintf("%1.3fms", res.GetDurationMs())
return Json(200, dtoRes)
}
// GET /api/alerts/:id
func GetAlert(c *middleware.Context) Response {
id := c.ParamsInt64(":alertId")
@@ -101,55 +139,53 @@ func DelAlert(c *middleware.Context) Response {
return Json(200, resp)
}
// GET /api/alerts/events/:id
func GetAlertStates(c *middleware.Context) Response {
alertId := c.ParamsInt64(":alertId")
query := models.GetAlertsStateQuery{
AlertId: alertId,
}
if err := bus.Dispatch(&query); err != nil {
return ApiError(500, "Failed get alert state log", err)
}
return Json(200, query.Result)
}
// PUT /api/alerts/events/:id
func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
cmd.AlertId = c.ParamsInt64(":alertId")
cmd.OrgId = c.OrgId
query := models.GetAlertByIdQuery{Id: cmd.AlertId}
if err := bus.Dispatch(&query); err != nil {
return ApiError(500, "Failed to get alertstate", err)
}
if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId {
return ApiError(500, "Alert not found", nil)
}
if err := bus.Dispatch(&cmd); err != nil {
return ApiError(500, "Failed to set new state", err)
}
return Json(200, cmd.Result)
}
// // GET /api/alerts/events/:id
// func GetAlertStates(c *middleware.Context) Response {
// alertId := c.ParamsInt64(":alertId")
//
// query := models.GetAlertsStateQuery{
// AlertId: alertId,
// }
//
// if err := bus.Dispatch(&query); err != nil {
// return ApiError(500, "Failed get alert state log", err)
// }
//
// return Json(200, query.Result)
// }
//
// // PUT /api/alerts/events/:id
// func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
// cmd.AlertId = c.ParamsInt64(":alertId")
// cmd.OrgId = c.OrgId
//
// query := models.GetAlertByIdQuery{Id: cmd.AlertId}
// if err := bus.Dispatch(&query); err != nil {
// return ApiError(500, "Failed to get alertstate", err)
// }
//
// if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId {
// return ApiError(500, "Alert not found", nil)
// }
//
// if err := bus.Dispatch(&cmd); err != nil {
// return ApiError(500, "Failed to set new state", err)
// }
//
// return Json(200, cmd.Result)
// }
func GetAlertNotifications(c *middleware.Context) Response {
query := &models.GetAlertNotificationQuery{
OrgID: c.OrgId,
}
query := &models.GetAlertNotificationsQuery{OrgId: c.OrgId}
if err := bus.Dispatch(query); err != nil {
return ApiError(500, "Failed to get alert notifications", err)
}
var result []dtos.AlertNotificationDTO
var result []dtos.AlertNotification
for _, notification := range query.Result {
result = append(result, dtos.AlertNotificationDTO{
result = append(result, dtos.AlertNotification{
Id: notification.Id,
Name: notification.Name,
Type: notification.Type,
@@ -162,8 +198,8 @@ func GetAlertNotifications(c *middleware.Context) Response {
}
func GetAlertNotificationById(c *middleware.Context) Response {
query := &models.GetAlertNotificationQuery{
OrgID: c.OrgId,
query := &models.GetAlertNotificationsQuery{
OrgId: c.OrgId,
Id: c.ParamsInt64("notificationId"),
}
@@ -175,7 +211,7 @@ func GetAlertNotificationById(c *middleware.Context) Response {
}
func CreateAlertNotification(c *middleware.Context, cmd models.CreateAlertNotificationCommand) Response {
cmd.OrgID = c.OrgId
cmd.OrgId = c.OrgId
if err := bus.Dispatch(&cmd); err != nil {
return ApiError(500, "Failed to create alert notification", err)
@@ -185,7 +221,7 @@ func CreateAlertNotification(c *middleware.Context, cmd models.CreateAlertNotifi
}
func UpdateAlertNotification(c *middleware.Context, cmd models.UpdateAlertNotificationCommand) Response {
cmd.OrgID = c.OrgId
cmd.OrgId = c.OrgId
if err := bus.Dispatch(&cmd); err != nil {
return ApiError(500, "Failed to update alert notification", err)
@@ -204,5 +240,5 @@ func DeleteAlertNotification(c *middleware.Context) Response {
return ApiError(500, "Failed to delete alert notification", err)
}
return Json(200, map[string]interface{}{"notificationId": cmd.Id})
return ApiSuccess("Notification deleted")
}
pkg/api/api.go
+2 -1
View File
@@ -246,7 +246,8 @@ func Register(r *macaron.Macaron) {
r.Get("/metrics", wrap(GetInternalMetrics))
r.Group("/alerts", func() {
r.Get("/:alertId/states", wrap(GetAlertStates))
r.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest))
//r.Get("/:alertId/states", wrap(GetAlertStates))
//r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState))
r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert))
//r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates
pkg/api/dtos/alerting.go
+37 -18
View File
@@ -1,31 +1,50 @@
package dtos
import "time"
import (
"time"
type AlertRuleDTO struct {
Id int64 `json:"id"`
DashboardId int64 `json:"dashboardId"`
PanelId int64 `json:"panelId"`
Query string `json:"query"`
QueryRefId string `json:"queryRefId"`
WarnLevel float64 `json:"warnLevel"`
CritLevel float64 `json:"critLevel"`
WarnOperator string `json:"warnOperator"`
CritOperator string `json:"critOperator"`
Frequency int64 `json:"frequency"`
Name string `json:"name"`
Description string `json:"description"`
QueryRange int `json:"queryRange"`
Aggregator string `json:"aggregator"`
State string `json:"state"`
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
)
type AlertRule struct {
Id int64 `json:"id"`
DashboardId int64 `json:"dashboardId"`
PanelId int64 `json:"panelId"`
Name string `json:"name"`
Description string `json:"description"`
State m.AlertStateType `json:"state"`
Severity m.AlertSeverityType `json:"severity"`
DashbboardUri string `json:"dashboardUri"`
}
type AlertNotificationDTO struct {
type AlertNotification struct {
Id int64 `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
type AlertTestCommand struct {
Dashboard *simplejson.Json `json:"dashboard" binding:"Required"`
PanelId int64 `json:"panelId" binding:"Required"`
}
type AlertTestResult struct {
Firing bool `json:"firing"`
TimeMs string `json:"timeMs"`
Error string `json:"error,omitempty"`
Logs []*AlertTestResultLog `json:"logs,omitempty"`
}
type AlertTestResultLog struct {
Message string `json:"message"`
Data interface{} `json:"data"`
}
type AlertEvent struct {
Metric string `json:"metric"`
Value float64 `json:"value"`
}
pkg/log/log.go
+3 -1
View File
@@ -116,7 +116,9 @@ func getFilters(filterStrArray []string) map[string]log15.Lvl {
for _, filterStr := range filterStrArray {
parts := strings.Split(filterStr, ":")
filterMap[parts[0]] = getLogLevelFromString(parts[1])
if len(parts) > 1 {
filterMap[parts[0]] = getLogLevelFromString(parts[1])
}
}
return filterMap
pkg/models/alert.go
+33 -29
View File
@@ -6,6 +6,29 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
)
type AlertStateType string
type AlertSeverityType string
const (
AlertStatePending AlertStateType = "pending"
AlertStateFiring AlertStateType = "firing"
AlertStateOK AlertStateType = "ok"
)
func (s AlertStateType) IsValid() bool {
return s == AlertStatePending || s == AlertStateFiring || s == AlertStateOK
}
const (
AlertSeverityCritical AlertSeverityType = "critical"
AlertSeverityWarning AlertSeverityType = "warning"
AlertSeverityInfo AlertSeverityType = "info"
)
func (s AlertSeverityType) IsValid() bool {
return s == AlertSeverityCritical || s == AlertSeverityInfo || s == AlertSeverityWarning
}
type Alert struct {
Id int64
OrgId int64
@@ -13,7 +36,8 @@ type Alert struct {
PanelId int64
Name string
Description string
State string
Severity AlertSeverityType
State AlertStateType
Handler int64
Enabled bool
Frequency int64
@@ -31,7 +55,7 @@ func (alert *Alert) ValidToSave() bool {
return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0
}
func (alert *Alert) ShouldUpdateState(newState string) bool {
func (alert *Alert) ShouldUpdateState(newState AlertStateType) bool {
return alert.State != newState
}
@@ -73,25 +97,6 @@ type HeartBeatCommand struct {
Result AlertingClusterInfo
}
type AlertChange struct {
Id int64 `json:"id"`
OrgId int64 `json:"-"`
AlertId int64 `json:"alertId"`
UpdatedBy int64 `json:"updatedBy"`
NewAlertSettings *simplejson.Json `json:"newAlertSettings"`
Type string `json:"type"`
Created time.Time `json:"created"`
}
// Commands
type CreateAlertChangeCommand struct {
OrgId int64
AlertId int64
UpdatedBy int64
NewAlertSettings *simplejson.Json
Type string
}
type SaveAlertsCommand struct {
DashboardId int64
UserId int64
@@ -100,6 +105,13 @@ type SaveAlertsCommand struct {
Alerts []*Alert
}
type SetAlertStateCommand struct {
AlertId int64
OrgId int64
State AlertStateType
Timestamp time.Time
}
type DeleteAlertCommand struct {
AlertId int64
}
@@ -123,11 +135,3 @@ type GetAlertByIdQuery struct {
Result *Alert
}
type GetAlertChangesQuery struct {
OrgId int64
Limit int64
SinceId int64
Result []*AlertChange
}
pkg/models/alert_notifications.go
+21 -25
View File
@@ -7,34 +7,31 @@ import (
)
type AlertNotification struct {
Id int64 `json:"id"`
OrgId int64 `json:"-"`
Name string `json:"name"`
Type string `json:"type"`
AlwaysExecute bool `json:"alwaysExecute"`
Settings *simplejson.Json `json:"settings"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Id int64 `json:"id"`
OrgId int64 `json:"-"`
Name string `json:"name"`
Type string `json:"type"`
Settings *simplejson.Json `json:"settings"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
type CreateAlertNotificationCommand struct {
Name string `json:"name" binding:"Required"`
Type string `json:"type" binding:"Required"`
AlwaysExecute bool `json:"alwaysExecute"`
OrgID int64 `json:"-"`
Settings *simplejson.Json `json:"settings"`
Name string `json:"name" binding:"Required"`
Type string `json:"type" binding:"Required"`
Settings *simplejson.Json `json:"settings"`
OrgId int64 `json:"-"`
Result *AlertNotification
}
type UpdateAlertNotificationCommand struct {
Id int64 `json:"id" binding:"Required"`
Name string `json:"name" binding:"Required"`
Type string `json:"type" binding:"Required"`
AlwaysExecute bool `json:"alwaysExecute"`
OrgID int64 `json:"-"`
Settings *simplejson.Json `json:"settings" binding:"Required"`
Id int64 `json:"id" binding:"Required"`
Name string `json:"name" binding:"Required"`
Type string `json:"type" binding:"Required"`
Settings *simplejson.Json `json:"settings" binding:"Required"`
OrgId int64 `json:"-"`
Result *AlertNotification
}
@@ -43,12 +40,11 @@ type DeleteAlertNotificationCommand struct {
OrgId int64
}
type GetAlertNotificationQuery struct {
Name string
Id int64
Ids []int64
OrgID int64
IncludeAlwaysExecute bool
type GetAlertNotificationsQuery struct {
Name string
Id int64
Ids []int64
OrgId int64
Result []*AlertNotification
}
pkg/models/alert_state.go
+45 -53
View File
@@ -1,55 +1,47 @@
package models
import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
)
type AlertState struct {
Id int64 `json:"-"`
OrgId int64 `json:"-"`
AlertId int64 `json:"alertId"`
State string `json:"state"`
Created time.Time `json:"created"`
Info string `json:"info"`
TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
}
func (this *UpdateAlertStateCommand) IsValidState() bool {
for _, v := range alertstates.ValidStates {
if this.State == v {
return true
}
}
return false
}
// Commands
type UpdateAlertStateCommand struct {
AlertId int64 `json:"alertId" binding:"Required"`
OrgId int64 `json:"orgId" binding:"Required"`
State string `json:"state" binding:"Required"`
Info string `json:"info"`
TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
Result *Alert
}
// Queries
type GetAlertsStateQuery struct {
OrgId int64 `json:"orgId" binding:"Required"`
AlertId int64 `json:"alertId" binding:"Required"`
Result *[]AlertState
}
type GetLastAlertStateQuery struct {
AlertId int64
OrgId int64
Result *AlertState
}
// type AlertState struct {
// Id int64 `json:"-"`
// OrgId int64 `json:"-"`
// AlertId int64 `json:"alertId"`
// State string `json:"state"`
// Created time.Time `json:"created"`
// Info string `json:"info"`
// TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
// }
//
// func (this *UpdateAlertStateCommand) IsValidState() bool {
// for _, v := range alertstates.ValidStates {
// if this.State == v {
// return true
// }
// }
// return false
// }
//
// // Commands
//
// type UpdateAlertStateCommand struct {
// AlertId int64 `json:"alertId" binding:"Required"`
// OrgId int64 `json:"orgId" binding:"Required"`
// State string `json:"state" binding:"Required"`
// Info string `json:"info"`
//
// Result *Alert
// }
//
// // Queries
//
// type GetAlertsStateQuery struct {
// OrgId int64 `json:"orgId" binding:"Required"`
// AlertId int64 `json:"alertId" binding:"Required"`
//
// Result *[]AlertState
// }
//
// type GetLastAlertStateQuery struct {
// AlertId int64
// OrgId int64
//
// Result *AlertState
// }
pkg/models/annotations.go
+22
View File
@@ -0,0 +1,22 @@
package models
import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
)
type AnnotationType string
type AnnotationEvent struct {
Id int64
OrgId int64
Type AnnotationType
Title string
Text string
AlertId int64
UserId int64
Timestamp time.Time
Data *simplejson.Json
}
pkg/services/alerting/alert_rule.go
+40 -62
View File
@@ -4,31 +4,32 @@ import (
"fmt"
"regexp"
"strconv"
"strings"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/alerting/transformers"
m "github.com/grafana/grafana/pkg/models"
)
type AlertRule struct {
Id int64
OrgId int64
DashboardId int64
PanelId int64
Frequency int64
Name string
Description string
State string
Warning Level
Critical Level
Query AlertQuery
Transform string
TransformParams simplejson.Json
Transformer transformers.Transformer
Id int64
OrgId int64
DashboardId int64
PanelId int64
Frequency int64
Name string
Description string
State m.AlertStateType
Severity m.AlertSeverityType
Conditions []AlertCondition
Notifications []int64
}
NotificationGroups []int64
type AlertValidationError struct {
Reason string
}
func (e AlertValidationError) Error() string {
return e.Reason
}
var (
@@ -59,60 +60,37 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
model := &AlertRule{}
model.Id = ruleDef.Id
model.OrgId = ruleDef.OrgId
model.DashboardId = ruleDef.DashboardId
model.PanelId = ruleDef.PanelId
model.Name = ruleDef.Name
model.Description = ruleDef.Description
model.State = ruleDef.State
model.Frequency = ruleDef.Frequency
model.Severity = ruleDef.Severity
model.State = ruleDef.State
ngs := ruleDef.Settings.Get("notificationGroups").MustString()
var ids []int64
for _, v := range strings.Split(ngs, ",") {
id, err := strconv.Atoi(v)
if err == nil {
ids = append(ids, int64(id))
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v)
if id, err := jsonModel.Get("id").Int64(); err != nil {
return nil, AlertValidationError{Reason: "Invalid notification schema"}
} else {
model.Notifications = append(model.Notifications, id)
}
}
model.NotificationGroups = ids
critical := ruleDef.Settings.Get("crit")
model.Critical = Level{
Operator: critical.Get("op").MustString(),
Value: critical.Get("value").MustFloat64(),
for index, condition := range ruleDef.Settings.Get("conditions").MustArray() {
conditionModel := simplejson.NewFromAny(condition)
switch conditionModel.Get("type").MustString() {
case "query":
queryCondition, err := NewQueryCondition(conditionModel, index)
if err != nil {
return nil, err
}
model.Conditions = append(model.Conditions, queryCondition)
}
}
warning := ruleDef.Settings.Get("warn")
model.Warning = Level{
Operator: warning.Get("op").MustString(),
Value: warning.Get("value").MustFloat64(),
}
model.Transform = ruleDef.Settings.Get("transform").Get("type").MustString()
if model.Transform == "" {
return nil, fmt.Errorf("missing transform")
}
model.TransformParams = *ruleDef.Settings.Get("transform")
if model.Transform == "aggregation" {
method := ruleDef.Settings.Get("transform").Get("method").MustString()
model.Transformer = transformers.NewAggregationTransformer(method)
}
query := ruleDef.Settings.Get("query")
model.Query = AlertQuery{
Query: query.Get("query").MustString(),
DatasourceId: query.Get("datasourceId").MustInt64(),
From: query.Get("from").MustString(),
To: query.Get("to").MustString(),
}
if model.Query.Query == "" {
return nil, fmt.Errorf("missing query.query")
}
if model.Query.DatasourceId == 0 {
return nil, fmt.Errorf("missing query.datasourceId")
if len(model.Conditions) == 0 {
return nil, fmt.Errorf("Alert is missing conditions")
}
return model, nil
pkg/services/alerting/alert_rule_test.go
+43 -25
View File
@@ -31,33 +31,30 @@ func TestAlertRuleModel(t *testing.T) {
So(seconds, ShouldEqual, 1)
})
Convey("", func() {
Convey("can construct alert rule model", func() {
json := `
{
"name": "name2",
"description": "desc2",
"handler": 0,
"enabled": true,
"crit": {
"value": 20,
"op": ">"
},
"warn": {
"value": 10,
"op": ">"
},
"frequency": "60s",
"query": {
"from": "5m",
"refId": "A",
"to": "now",
"query": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)",
"datasourceId": 1
},
"transform": {
"type": "avg",
"name": "aggregation"
}
"conditions": [
{
"type": "query",
"query": {
"params": ["A", "5m", "now"],
"datasourceId": 1,
"model": {"target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"}
},
"reducer": {"type": "avg", "params": []},
"evaluator": {"type": ">", "params": [100]}
}
],
"notifications": [
{"id": 1134},
{"id": 22}
]
}
`
@@ -72,15 +69,36 @@ func TestAlertRuleModel(t *testing.T) {
Settings: alertJSON,
}
alertRule, err := NewAlertRuleFromDBModel(alert)
alertRule, err := NewAlertRuleFromDBModel(alert)
So(err, ShouldBeNil)
So(alertRule.Warning.Operator, ShouldEqual, ">")
So(alertRule.Warning.Value, ShouldEqual, 10)
So(alertRule.Conditions, ShouldHaveLength, 1)
So(alertRule.Critical.Operator, ShouldEqual, ">")
So(alertRule.Critical.Value, ShouldEqual, 20)
Convey("Can read query condition from json model", func() {
queryCondition, ok := alertRule.Conditions[0].(*QueryCondition)
So(ok, ShouldBeTrue)
So(queryCondition.Query.From, ShouldEqual, "5m")
So(queryCondition.Query.To, ShouldEqual, "now")
So(queryCondition.Query.DatasourceId, ShouldEqual, 1)
Convey("Can read query reducer", func() {
reducer, ok := queryCondition.Reducer.(*SimpleReducer)
So(ok, ShouldBeTrue)
So(reducer.Type, ShouldEqual, "avg")
})
Convey("Can read evaluator", func() {
evaluator, ok := queryCondition.Evaluator.(*DefaultAlertEvaluator)
So(ok, ShouldBeTrue)
So(evaluator.Type, ShouldEqual, ">")
})
})
Convey("Can read notifications", func() {
So(len(alertRule.Notifications), ShouldEqual, 2)
})
})
})
}
pkg/services/alerting/alertstates/states.go
-16
View File
@@ -1,16 +0,0 @@
package alertstates
var (
ValidStates = []string{
Ok,
Warn,
Critical,
Unknown,
}
Ok = "OK"
Warn = "WARN"
Critical = "CRITICAL"
Pending = "PENDING"
Unknown = "UNKNOWN"
)
pkg/services/alerting/conditions.go
+191
View File
@@ -0,0 +1,191 @@
package alerting
import (
"encoding/json"
"fmt"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type QueryCondition struct {
Index int
Query AlertQuery
Reducer QueryReducer
Evaluator AlertEvaluator
HandleRequest tsdb.HandleRequestFunc
}
func (c *QueryCondition) Eval(context *AlertResultContext) {
seriesList, err := c.executeQuery(context)
if err != nil {
context.Error = err
return
}
for _, series := range seriesList {
reducedValue := c.Reducer.Reduce(series)
pass := c.Evaluator.Eval(series, reducedValue)
if context.IsTestRun {
context.Logs = append(context.Logs, &AlertResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, pass, series.Name, reducedValue),
})
}
if pass {
context.Events = append(context.Events, &AlertEvent{
Metric: series.Name,
Value: reducedValue,
})
context.Firing = true
break
}
}
}
func (c *QueryCondition) executeQuery(context *AlertResultContext) (tsdb.TimeSeriesSlice, error) {
getDsInfo := &m.GetDataSourceByIdQuery{
Id: c.Query.DatasourceId,
OrgId: context.Rule.OrgId,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource")
}
req := c.getRequestForAlertRule(getDsInfo.Result)
result := make(tsdb.TimeSeriesSlice, 0)
resp, err := c.HandleRequest(req)
if err != nil {
return nil, fmt.Errorf("tsdb.HandleRequest() error %v", err)
}
for _, v := range resp.Results {
if v.Error != nil {
return nil, fmt.Errorf("tsdb.HandleRequest() response error %v", v)
}
result = append(result, v.Series...)
if context.IsTestRun {
context.Logs = append(context.Logs, &AlertResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Query Result", c.Index),
Data: v.Series,
})
}
}
return result, nil
}
func (c *QueryCondition) getRequestForAlertRule(datasource *m.DataSource) *tsdb.Request {
req := &tsdb.Request{
TimeRange: tsdb.TimeRange{
From: c.Query.From,
To: c.Query.To,
},
Queries: []*tsdb.Query{
{
RefId: "A",
Query: c.Query.Model.Get("target").MustString(),
DataSource: &tsdb.DataSourceInfo{
Id: datasource.Id,
Name: datasource.Name,
PluginId: datasource.Type,
Url: datasource.Url,
},
},
},
}
return req
}
func NewQueryCondition(model *simplejson.Json, index int) (*QueryCondition, error) {
condition := QueryCondition{}
condition.Index = index
condition.HandleRequest = tsdb.HandleRequest
queryJson := model.Get("query")
condition.Query.Model = queryJson.Get("model")
condition.Query.From = queryJson.Get("params").MustArray()[1].(string)
condition.Query.To = queryJson.Get("params").MustArray()[2].(string)
condition.Query.DatasourceId = queryJson.Get("datasourceId").MustInt64()
reducerJson := model.Get("reducer")
condition.Reducer = NewSimpleReducer(reducerJson.Get("type").MustString())
evaluatorJson := model.Get("evaluator")
evaluator, err := NewDefaultAlertEvaluator(evaluatorJson)
if err != nil {
return nil, err
}
condition.Evaluator = evaluator
return &condition, nil
}
type SimpleReducer struct {
Type string
}
func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) float64 {
var value float64 = 0
switch s.Type {
case "avg":
for _, point := range series.Points {
value += point[0]
}
value = value / float64(len(series.Points))
}
return value
}
func NewSimpleReducer(typ string) *SimpleReducer {
return &SimpleReducer{Type: typ}
}
type DefaultAlertEvaluator struct {
Type string
Threshold float64
}
func (e *DefaultAlertEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
switch e.Type {
case ">":
return reducedValue > e.Threshold
case "<":
return reducedValue < e.Threshold
}
return false
}
func NewDefaultAlertEvaluator(model *simplejson.Json) (*DefaultAlertEvaluator, error) {
evaluator := &DefaultAlertEvaluator{}
evaluator.Type = model.Get("type").MustString()
if evaluator.Type == "" {
return nil, AlertValidationError{Reason: "Evaluator missing type property"}
}
params := model.Get("params").MustArray()
if len(params) == 0 {
return nil, AlertValidationError{Reason: "Evaluator missing threshold parameter"}
}
threshold, ok := params[0].(json.Number)
if !ok {
return nil, AlertValidationError{Reason: "Evaluator has invalid threshold parameter"}
}
evaluator.Threshold, _ = threshold.Float64()
return evaluator, nil
}
pkg/services/alerting/conditions_test.go
+92
View File
@@ -0,0 +1,92 @@
package alerting
import (
"testing"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
. "github.com/smartystreets/goconvey/convey"
)
func TestQueryCondition(t *testing.T) {
Convey("when evaluating query condition", t, func() {
queryConditionScenario("Given avg() and > 100", func(ctx *queryConditionTestContext) {
ctx.reducer = `{"type": "avg"}`
ctx.evaluator = `{"type": ">", "params": [100]}`
Convey("should fire when avg is above 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Firing, ShouldBeTrue)
})
Convey("Should not fire when avg is below 100", func() {
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})}
ctx.exec()
So(ctx.result.Error, ShouldBeNil)
So(ctx.result.Firing, ShouldBeFalse)
})
})
})
}
type queryConditionTestContext struct {
reducer string
evaluator string
series tsdb.TimeSeriesSlice
result *AlertResultContext
}
type queryConditionScenarioFunc func(c *queryConditionTestContext)
func (ctx *queryConditionTestContext) exec() {
jsonModel, err := simplejson.NewJson([]byte(`{
"type": "query",
"query": {
"params": ["A", "5m", "now"],
"datasourceId": 1,
"model": {"target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"}
},
"reducer":` + ctx.reducer + `,
"evaluator":` + ctx.evaluator + `
}`))
So(err, ShouldBeNil)
condition, err := NewQueryCondition(jsonModel, 0)
So(err, ShouldBeNil)
condition.HandleRequest = func(req *tsdb.Request) (*tsdb.Response, error) {
return &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"A": {Series: ctx.series},
},
}, nil
}
condition.Eval(ctx.result)
}
func queryConditionScenario(desc string, fn queryConditionScenarioFunc) {
Convey(desc, func() {
bus.AddHandler("test", func(query *m.GetDataSourceByIdQuery) error {
query.Result = &m.DataSource{Id: 1, Type: "graphite"}
return nil
})
ctx := &queryConditionTestContext{}
ctx.result = &AlertResultContext{
Rule: &AlertRule{},
}
fn(ctx)
})
}
pkg/services/alerting/engine.go
+18 -52
View File
@@ -1,38 +1,34 @@
package alerting
import (
"fmt"
"time"
"github.com/benbjohnson/clock"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
)
type Engine struct {
execQueue chan *AlertJob
resultQueue chan *AlertResult
resultQueue chan *AlertResultContext
clock clock.Clock
ticker *Ticker
scheduler Scheduler
handler AlertingHandler
handler AlertHandler
ruleReader RuleReader
log log.Logger
responseHandler ResultHandler
alertJobTimeout time.Duration
}
func NewEngine() *Engine {
e := &Engine{
ticker: NewTicker(time.Now(), time.Second*0, clock.New()),
execQueue: make(chan *AlertJob, 1000),
resultQueue: make(chan *AlertResult, 1000),
resultQueue: make(chan *AlertResultContext, 1000),
scheduler: NewScheduler(),
handler: NewHandler(),
ruleReader: NewRuleReader(),
log: log.New("alerting.engine"),
responseHandler: NewResultHandler(),
alertJobTimeout: time.Second * 5,
}
return e
@@ -75,41 +71,25 @@ func (e *Engine) alertingTicker() {
}
func (e *Engine) execDispatch() {
defer func() {
if err := recover(); err != nil {
e.log.Error("Scheduler Panic: stopping executor", "error", err, "stack", log.Stack(1))
}
}()
for job := range e.execQueue {
log.Trace("Alerting: engine:execDispatch() starting job %s", job.Rule.Name)
job.Running = true
e.executeJob(job)
e.log.Debug("Starting executing alert rule %s", job.Rule.Name)
go e.executeJob(job)
}
}
func (e *Engine) executeJob(job *AlertJob) {
startTime := time.Now()
resultChan := make(chan *AlertResult, 1)
go e.handler.Execute(job, resultChan)
select {
case <-time.After(e.alertJobTimeout):
e.resultQueue <- &AlertResult{
State: alertstates.Pending,
Error: fmt.Errorf("Timeout"),
AlertJob: job,
StartTime: startTime,
EndTime: time.Now(),
defer func() {
if err := recover(); err != nil {
e.log.Error("Execute Alert Panic", "error", err, "stack", log.Stack(1))
}
close(resultChan)
e.log.Debug("Job Execution timeout", "alertRuleId", job.Rule.Id)
case result := <-resultChan:
duration := float64(result.EndTime.Nanosecond()-result.StartTime.Nanosecond()) / float64(1000000)
e.log.Debug("Job Execution done", "timeTakenMs", duration, "ruleId", job.Rule.Id)
e.resultQueue <- result
}
}()
job.Running = true
context := NewAlertResultContext(job.Rule)
e.handler.Execute(context)
job.Running = false
e.resultQueue <- context
}
func (e *Engine) resultHandler() {
@@ -120,25 +100,11 @@ func (e *Engine) resultHandler() {
}()
for result := range e.resultQueue {
e.log.Debug("Alert Rule Result", "ruleId", result.AlertJob.Rule.Id, "state", result.State, "retry", result.AlertJob.RetryCount)
result.AlertJob.Running = false
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
if result.Error != nil {
result.AlertJob.IncRetry()
if result.AlertJob.Retryable() {
e.log.Error("Alert Rule Result Error", "ruleId", result.AlertJob.Rule.Id, "error", result.Error, "retry", result.AlertJob.RetryCount)
e.execQueue <- result.AlertJob
} else {
e.log.Error("Alert Rule Result Error After Max Retries", "ruleId", result.AlertJob.Rule.Id, "error", result.Error, "retry", result.AlertJob.RetryCount)
result.State = alertstates.Critical
result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error)
e.responseHandler.Handle(result)
}
e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry")
} else {
result.AlertJob.ResetRetry()
e.responseHandler.Handle(result)
}
}
pkg/services/alerting/extractor.go
+38 -23
View File
@@ -47,6 +47,17 @@ func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (*m.DataSource, e
return nil, errors.New("Could not find datasource id for " + dsName)
}
func findPanelQueryByRefId(panel *simplejson.Json, refId string) *simplejson.Json {
for _, targetsObj := range panel.Get("targets").MustArray() {
target := simplejson.NewFromAny(targetsObj)
if target.Get("refId").MustString() == refId {
return target
}
}
return nil
}
func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
e.log.Debug("GetAlerts")
@@ -78,34 +89,39 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
Handler: jsonAlert.Get("handler").MustInt64(),
Enabled: jsonAlert.Get("enabled").MustBool(),
Description: jsonAlert.Get("description").MustString(),
Severity: m.AlertSeverityType(jsonAlert.Get("severity").MustString()),
Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()),
}
valueQuery := jsonAlert.Get("query")
valueQueryRef := valueQuery.Get("refId").MustString()
for _, targetsObj := range panel.Get("targets").MustArray() {
target := simplejson.NewFromAny(targetsObj)
if !alert.Severity.IsValid() {
return nil, AlertValidationError{Reason: "Invalid alert Severity"}
}
if target.Get("refId").MustString() == valueQueryRef {
dsName := ""
if target.Get("datasource").MustString() != "" {
dsName = target.Get("datasource").MustString()
} else if panel.Get("datasource").MustString() != "" {
dsName = panel.Get("datasource").MustString()
}
for _, condition := range jsonAlert.Get("conditions").MustArray() {
jsonCondition := simplejson.NewFromAny(condition)
if datasource, err := e.lookupDatasourceId(dsName); err != nil {
return nil, err
} else {
valueQuery.SetPath([]string{"datasourceId"}, datasource.Id)
valueQuery.SetPath([]string{"datasourceType"}, datasource.Type)
}
jsonQuery := jsonCondition.Get("query")
queryRefId := jsonQuery.Get("params").MustArray()[0].(string)
panelQuery := findPanelQueryByRefId(panel, queryRefId)
targetQuery := target.Get("target").MustString()
if targetQuery != "" {
jsonAlert.SetPath([]string{"query", "query"}, targetQuery)
}
if panelQuery == nil {
return nil, AlertValidationError{Reason: "Alert refes to query that cannot be found"}
}
dsName := ""
if panelQuery.Get("datasource").MustString() != "" {
dsName = panelQuery.Get("datasource").MustString()
} else if panel.Get("datasource").MustString() != "" {
dsName = panel.Get("datasource").MustString()
}
if datasource, err := e.lookupDatasourceId(dsName); err != nil {
return nil, err
} else {
jsonQuery.SetPath([]string{"datasourceId"}, datasource.Id)
}
jsonQuery.Set("model", panelQuery.Interface())
}
alert.Settings = jsonAlert
@@ -116,9 +132,8 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
alerts = append(alerts, alert)
} else {
e.log.Error("Failed to extract alerts from dashboard", "error", err)
return nil, errors.New("Failed to extract alerts from dashboard")
return nil, err
}
}
}
pkg/services/alerting/extractor_test.go
+73 -147
View File
@@ -14,162 +14,71 @@ func TestAlertRuleExtraction(t *testing.T) {
Convey("Parsing alert rules from dashboard json", t, func() {
Convey("Parsing and validating alerts from dashboards", func() {
json := `{
"id": 57,
"title": "Graphite 4",
"originalTitle": "Graphite 4",
"tags": [
"graphite"
],
"rows": [
{
"panels": [
"id": 57,
"title": "Graphite 4",
"originalTitle": "Graphite 4",
"tags": ["graphite"],
"rows": [
{
"title": "Active desktop users",
"editable": true,
"type": "graph",
"id": 3,
"targets": [
"panels": [
{
"title": "Active desktop users",
"editable": true,
"type": "graph",
"id": 3,
"targets": [
{
"refId": "A",
"target": "aliasByNode(statsd.fakesite.counters.session_start.desktop.count, 4)"
}
],
"datasource": null,
"alert": {
"name": "name1",
"description": "desc1",
"handler": 1,
"enabled": true,
"critical": {
"value": 20,
"op": ">"
},
"frequency": "60s",
"query": {
"from": "5m",
"refId": "A",
"to": "now"
},
"transform": {
"type": "avg",
"name": "aggregation"
},
"warn": {
"value": 10,
"op": ">"
}
}
},
{
"title": "Active mobile users",
"id": 4,
"targets": [
{
"refId": "A",
"target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"
}
],
"datasource": "graphite2",
"alert": {
"name": "name2",
"description": "desc2",
"handler": 0,
"enabled": true,
"critical": {
"value": 20,
"op": ">"
},
"frequency": "60s",
"query": {
"from": "5m",
"refId": "A",
"to": "now"
},
"transform": {
"type": "avg",
"name": "aggregation"
},
"warn": {
"value": 10,
"op": ">"
}
}
}
],
"title": "Row"
},
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"datasource": "InfluxDB",
"id": 2,
"alert": {
"name": "name2",
"description": "desc2",
"enabled": false,
"critical": {
"level": 20,
"op": ">"
},
"warn": {
"level": 10,
"op": ">"
],
"datasource": null,
"alert": {
"name": "name1",
"description": "desc1",
"handler": 1,
"enabled": true,
"frequency": "60s",
"severity": "critical",
"conditions": [
{
"type": "query",
"query": {"params": ["A", "5m", "now"]},
"reducer": {"type": "avg", "params": []},
"evaluator": {"type": ">", "params": [100]}
}
]
}
},
"targets": [
{
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "cpu",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"cpu\" WHERE $timeFilter GROUP BY time($interval) fill(null)",
"refId": "A",
"resultFormat": "table",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": [],
"target": ""
{
"title": "Active mobile users",
"id": 4,
"targets": [
{"refId": "A", "target": ""},
{"refId": "B", "target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"}
],
"datasource": "graphite2",
"alert": {
"name": "name2",
"description": "desc2",
"handler": 0,
"enabled": true,
"frequency": "60s",
"severity": "warning",
"conditions": [
{
"type": "query",
"query": {"params": ["B", "5m", "now"]},
"reducer": {"type": "avg", "params": []},
"evaluator": {"type": ">", "params": [100]}
}
]
}
],
"title": "Broken influxdb panel",
"transform": "table",
"type": "table"
}
]
}
],
"title": "New row"
}
]
}`
]
}`
dashJson, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
@@ -215,6 +124,11 @@ func TestAlertRuleExtraction(t *testing.T) {
So(alerts[1].Handler, ShouldEqual, 0)
})
Convey("should extract Severity property", func() {
So(alerts[0].Severity, ShouldEqual, "critical")
So(alerts[1].Severity, ShouldEqual, "warning")
})
Convey("should extract frequency in seconds", func() {
So(alerts[0].Frequency, ShouldEqual, 60)
So(alerts[1].Frequency, ShouldEqual, 60)
@@ -231,6 +145,18 @@ func TestAlertRuleExtraction(t *testing.T) {
So(alerts[1].Name, ShouldEqual, "name2")
So(alerts[1].Description, ShouldEqual, "desc2")
})
Convey("should set datasourceId", func() {
condition := simplejson.NewFromAny(alerts[0].Settings.Get("conditions").MustArray()[0])
query := condition.Get("query")
So(query.Get("datasourceId").MustInt64(), ShouldEqual, 12)
})
Convey("should copy query model to condition", func() {
condition := simplejson.NewFromAny(alerts[0].Settings.Get("conditions").MustArray()[0])
model := condition.Get("query").Get("model")
So(model.Get("target").MustString(), ShouldEqual, "aliasByNode(statsd.fakesite.counters.session_start.desktop.count, 4)")
})
})
})
})
pkg/services/alerting/handler.go
+125 -114
View File
@@ -4,11 +4,7 @@ import (
"fmt"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
"github.com/grafana/grafana/pkg/tsdb"
)
var (
@@ -16,133 +12,148 @@ var (
)
type HandlerImpl struct {
log log.Logger
log log.Logger
alertJobTimeout time.Duration
}
func NewHandler() *HandlerImpl {
return &HandlerImpl{
log: log.New("alerting.executor"),
log: log.New("alerting.handler"),
alertJobTimeout: time.Second * 5,
}
}
func (e *HandlerImpl) Execute(job *AlertJob, resultQueue chan *AlertResult) {
startTime := time.Now()
func (e *HandlerImpl) Execute(context *AlertResultContext) {
timeSeries, err := e.executeQuery(job)
if err != nil {
resultQueue <- &AlertResult{
Error: err,
State: alertstates.Pending,
AlertJob: job,
StartTime: time.Now(),
EndTime: time.Now(),
}
go e.eval(context)
select {
case <-time.After(e.alertJobTimeout):
context.Error = fmt.Errorf("Timeout")
context.EndTime = time.Now()
e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
case <-context.DoneChan:
e.log.Debug("Job Execution done", "timeMs", context.GetDurationMs(), "alertId", context.Rule.Id, "firing", context.Firing)
}
result := e.evaluateRule(job.Rule, timeSeries)
result.AlertJob = job
result.StartTime = startTime
result.EndTime = time.Now()
resultQueue <- result
}
func (e *HandlerImpl) executeQuery(job *AlertJob) (tsdb.TimeSeriesSlice, error) {
getDsInfo := &m.GetDataSourceByIdQuery{
Id: job.Rule.Query.DatasourceId,
OrgId: job.Rule.OrgId,
}
func (e *HandlerImpl) eval(context *AlertResultContext) {
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource")
}
for _, condition := range context.Rule.Conditions {
condition.Eval(context)
req := e.GetRequestForAlertRule(job.Rule, getDsInfo.Result)
result := make(tsdb.TimeSeriesSlice, 0)
resp, err := tsdb.HandleRequest(req)
if err != nil {
return nil, fmt.Errorf("Alerting: GetSeries() tsdb.HandleRequest() error %v", err)
}
for _, v := range resp.Results {
if v.Error != nil {
return nil, fmt.Errorf("Alerting: GetSeries() tsdb.HandleRequest() response error %v", v)
// break if condition could not be evaluated
if context.Error != nil {
break
}
result = append(result, v.Series...)
// break if result has not triggered yet
if context.Firing == false {
break
}
}
return result, nil
context.EndTime = time.Now()
context.DoneChan <- true
}
func (e *HandlerImpl) GetRequestForAlertRule(rule *AlertRule, datasource *m.DataSource) *tsdb.Request {
e.log.Debug("GetRequest", "query", rule.Query.Query, "from", rule.Query.From, "datasourceId", datasource.Id)
req := &tsdb.Request{
TimeRange: tsdb.TimeRange{
From: "-" + rule.Query.From,
To: rule.Query.To,
},
Queries: []*tsdb.Query{
{
RefId: "A",
Query: rule.Query.Query,
DataSource: &tsdb.DataSourceInfo{
Id: datasource.Id,
Name: datasource.Name,
PluginId: datasource.Type,
Url: datasource.Url,
},
},
},
}
return req
}
func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) *AlertResult {
e.log.Debug("Evaluating Alerting Rule", "seriesCount", len(series), "ruleName", rule.Name)
triggeredAlert := make([]*TriggeredAlert, 0)
for _, serie := range series {
e.log.Debug("Evaluating series", "series", serie.Name)
transformedValue, _ := rule.Transformer.Transform(serie)
critResult := evalCondition(rule.Critical, transformedValue)
condition2 := fmt.Sprintf("%v %s %v ", transformedValue, rule.Critical.Operator, rule.Critical.Value)
e.log.Debug("Alert execution Crit", "name", serie.Name, "condition", condition2, "result", critResult)
if critResult {
triggeredAlert = append(triggeredAlert, &TriggeredAlert{
State: alertstates.Critical,
Value: transformedValue,
Metric: serie.Name,
})
continue
}
warnResult := evalCondition(rule.Warning, transformedValue)
condition := fmt.Sprintf("%v %s %v ", transformedValue, rule.Warning.Operator, rule.Warning.Value)
e.log.Debug("Alert execution Warn", "name", serie.Name, "condition", condition, "result", warnResult)
if warnResult {
triggeredAlert = append(triggeredAlert, &TriggeredAlert{
State: alertstates.Warn,
Value: transformedValue,
Metric: serie.Name,
})
}
}
executionState := alertstates.Ok
for _, raised := range triggeredAlert {
if raised.State == alertstates.Critical {
executionState = alertstates.Critical
}
if executionState != alertstates.Critical && raised.State == alertstates.Warn {
executionState = alertstates.Warn
}
}
return &AlertResult{State: executionState, TriggeredAlerts: triggeredAlert}
}
// func (e *HandlerImpl) executeQuery(job *AlertJob) (tsdb.TimeSeriesSlice, error) {
// getDsInfo := &m.GetDataSourceByIdQuery{
// Id: job.Rule.Query.DatasourceId,
// OrgId: job.Rule.OrgId,
// }
//
// if err := bus.Dispatch(getDsInfo); err != nil {
// return nil, fmt.Errorf("Could not find datasource")
// }
//
// req := e.GetRequestForAlertRule(job.Rule, getDsInfo.Result)
// result := make(tsdb.TimeSeriesSlice, 0)
//
// resp, err := tsdb.HandleRequest(req)
// if err != nil {
// return nil, fmt.Errorf("Alerting: GetSeries() tsdb.HandleRequest() error %v", err)
// }
//
// for _, v := range resp.Results {
// if v.Error != nil {
// return nil, fmt.Errorf("Alerting: GetSeries() tsdb.HandleRequest() response error %v", v)
// }
//
// result = append(result, v.Series...)
// }
//
// return result, nil
// }
//
// func (e *HandlerImpl) GetRequestForAlertRule(rule *AlertRule, datasource *m.DataSource) *tsdb.Request {
// e.log.Debug("GetRequest", "query", rule.Query.Query, "from", rule.Query.From, "datasourceId", datasource.Id)
// req := &tsdb.Request{
// TimeRange: tsdb.TimeRange{
// From: "-" + rule.Query.From,
// To: rule.Query.To,
// },
// Queries: []*tsdb.Query{
// {
// RefId: "A",
// Query: rule.Query.Query,
// DataSource: &tsdb.DataSourceInfo{
// Id: datasource.Id,
// Name: datasource.Name,
// PluginId: datasource.Type,
// Url: datasource.Url,
// },
// },
// },
// }
//
// return req
// }
//
// func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) *AlertResult {
// e.log.Debug("Evaluating Alerting Rule", "seriesCount", len(series), "ruleName", rule.Name)
//
// triggeredAlert := make([]*TriggeredAlert, 0)
//
// for _, serie := range series {
// e.log.Debug("Evaluating series", "series", serie.Name)
// transformedValue, _ := rule.Transformer.Transform(serie)
//
// critResult := evalCondition(rule.Critical, transformedValue)
// condition2 := fmt.Sprintf("%v %s %v ", transformedValue, rule.Critical.Operator, rule.Critical.Value)
// e.log.Debug("Alert execution Crit", "name", serie.Name, "condition", condition2, "result", critResult)
// if critResult {
// triggeredAlert = append(triggeredAlert, &TriggeredAlert{
// State: alertstates.Critical,
// Value: transformedValue,
// Metric: serie.Name,
// })
// continue
// }
//
// warnResult := evalCondition(rule.Warning, transformedValue)
// condition := fmt.Sprintf("%v %s %v ", transformedValue, rule.Warning.Operator, rule.Warning.Value)
// e.log.Debug("Alert execution Warn", "name", serie.Name, "condition", condition, "result", warnResult)
// if warnResult {
// triggeredAlert = append(triggeredAlert, &TriggeredAlert{
// State: alertstates.Warn,
// Value: transformedValue,
// Metric: serie.Name,
// })
// }
// }
//
// executionState := alertstates.Ok
// for _, raised := range triggeredAlert {
// if raised.State == alertstates.Critical {
// executionState = alertstates.Critical
// }
//
// if executionState != alertstates.Critical && raised.State == alertstates.Warn {
// executionState = alertstates.Warn
// }
// }
//
// return &AlertResult{State: executionState, TriggeredAlerts: triggeredAlert}
// }
pkg/services/alerting/handler_test.go
+144 -131
View File
@@ -3,149 +3,162 @@ package alerting
import (
"testing"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
"github.com/grafana/grafana/pkg/services/alerting/transformers"
"github.com/grafana/grafana/pkg/tsdb"
. "github.com/smartystreets/goconvey/convey"
)
type conditionStub struct {
firing bool
}
func (c *conditionStub) Eval(context *AlertResultContext) {
context.Firing = c.firing
}
func TestAlertingExecutor(t *testing.T) {
Convey("Test alert execution", t, func() {
executor := NewHandler()
handler := NewHandler()
Convey("single time serie", func() {
Convey("Show return ok since avg is above 2", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Ok)
})
Convey("Show return critical since below 2", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: "<"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Critical)
})
Convey("Show return critical since sum is above 10", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("sum"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{9, 0}, {9, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Critical)
})
Convey("Show return ok since avg is below 10", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{9, 0}, {9, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Ok)
})
Convey("Show return ok since min is below 10", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}, {9, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Ok)
})
Convey("Show return ok since max is above 10", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("max"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{6, 0}, {11, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Critical)
Convey("Show return triggered with single passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{&conditionStub{
firing: true,
}},
})
handler.eval(context)
So(context.Firing, ShouldEqual, true)
})
Convey("muliple time series", func() {
Convey("both are ok", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Ok)
Convey("Show return false with not passing condition", func() {
context := NewAlertResultContext(&AlertRule{
Conditions: []AlertCondition{
&conditionStub{firing: true},
&conditionStub{firing: false},
},
})
Convey("first serie is good, second is critical", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Critical)
})
Convey("first serie is warn, second is critical", func() {
rule := &AlertRule{
Critical: Level{Value: 10, Operator: ">"},
Warning: Level{Value: 5, Operator: ">"},
Transformer: transformers.NewAggregationTransformer("avg"),
}
timeSeries := []*tsdb.TimeSeries{
tsdb.NewTimeSeries("test1", [][2]float64{{6, 0}}),
tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}}),
}
result := executor.evaluateRule(rule, timeSeries)
So(result.State, ShouldEqual, alertstates.Critical)
})
handler.eval(context)
So(context.Firing, ShouldEqual, false)
})
// Convey("Show return critical since below 2", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: "<"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Critical)
// })
//
// Convey("Show return critical since sum is above 10", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("sum"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{9, 0}, {9, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Critical)
// })
//
// Convey("Show return ok since avg is below 10", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{9, 0}, {9, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Ok)
// })
//
// Convey("Show return ok since min is below 10", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}, {9, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Ok)
// })
//
// Convey("Show return ok since max is above 10", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("max"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{6, 0}, {11, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Critical)
// })
//
// })
//
// Convey("muliple time series", func() {
// Convey("both are ok", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
// tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Ok)
// })
//
// Convey("first serie is good, second is critical", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{2, 0}}),
// tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Critical)
// })
//
// Convey("first serie is warn, second is critical", func() {
// rule := &AlertRule{
// Critical: Level{Value: 10, Operator: ">"},
// Warning: Level{Value: 5, Operator: ">"},
// Transformer: transformers.NewAggregationTransformer("avg"),
// }
//
// timeSeries := []*tsdb.TimeSeries{
// tsdb.NewTimeSeries("test1", [][2]float64{{6, 0}}),
// tsdb.NewTimeSeries("test1", [][2]float64{{11, 0}}),
// }
//
// result := executor.evaluateRule(rule, timeSeries)
// So(result.State, ShouldEqual, alertstates.Critical)
// })
// })
})
}
pkg/services/alerting/interfaces.go
+21 -4
View File
@@ -1,9 +1,13 @@
package alerting
import "time"
import (
"time"
type AlertingHandler interface {
Execute(rule *AlertJob, resultChan chan *AlertResult)
"github.com/grafana/grafana/pkg/tsdb"
)
type AlertHandler interface {
Execute(context *AlertResultContext)
}
type Scheduler interface {
@@ -12,5 +16,18 @@ type Scheduler interface {
}
type Notifier interface {
Notify(alertResult *AlertResult)
Notify(alertResult *AlertResultContext)
GetType() string
}
type AlertCondition interface {
Eval(result *AlertResultContext)
}
type QueryReducer interface {
Reduce(timeSeries *tsdb.TimeSeries) float64
}
type AlertEvaluator interface {
Eval(timeSeries *tsdb.TimeSeries, reducedValue float64) bool
}
pkg/services/alerting/models.go
+42 -12
View File
@@ -1,6 +1,11 @@
package alerting
import "time"
import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
)
type AlertJob struct {
Offset int64
@@ -22,18 +27,43 @@ func (aj *AlertJob) IncRetry() {
aj.RetryCount++
}
type AlertResult struct {
State string
TriggeredAlerts []*TriggeredAlert
Error error
Description string
StartTime time.Time
EndTime time.Time
AlertJob *AlertJob
type AlertResultContext struct {
Firing bool
IsTestRun bool
Events []*AlertEvent
Logs []*AlertResultLogEntry
Error error
Description string
StartTime time.Time
EndTime time.Time
Rule *AlertRule
DoneChan chan bool
CancelChan chan bool
log log.Logger
}
type TriggeredAlert struct {
func (a *AlertResultContext) GetDurationMs() float64 {
return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
}
func NewAlertResultContext(rule *AlertRule) *AlertResultContext {
return &AlertResultContext{
StartTime: time.Now(),
Rule: rule,
Logs: make([]*AlertResultLogEntry, 0),
Events: make([]*AlertEvent, 0),
DoneChan: make(chan bool, 1),
CancelChan: make(chan bool, 1),
log: log.New("alerting.engine"),
}
}
type AlertResultLogEntry struct {
Message string
Data interface{}
}
type AlertEvent struct {
Value float64
Metric string
State string
@@ -46,7 +76,7 @@ type Level struct {
}
type AlertQuery struct {
Query string
Model *simplejson.Json
DatasourceId int64
From string
To string
pkg/services/alerting/notifier.go
+115 -162
View File
@@ -1,207 +1,160 @@
package alerting
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
"github.com/grafana/grafana/pkg/setting"
)
type NotifierImpl struct {
log log.Logger
getNotifications func(orgId int64, notificationGroups []int64) []*Notification
}
func NewNotifier() *NotifierImpl {
log := log.New("alerting.notifier")
return &NotifierImpl{
log: log,
getNotifications: buildGetNotifiers(log),
}
}
func (n NotifierImpl) ShouldDispath(alertResult *AlertResult, notifier *Notification) bool {
warn := alertResult.State == alertstates.Warn && notifier.SendWarning
crit := alertResult.State == alertstates.Critical && notifier.SendCritical
return (warn || crit) || alertResult.State == alertstates.Ok
}
func (n *NotifierImpl) Notify(alertResult *AlertResult) {
notifiers := n.getNotifications(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups)
for _, notifier := range notifiers {
if n.ShouldDispath(alertResult, notifier) {
n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type)
go notifier.Notifierr.Dispatch(alertResult)
}
}
}
type Notification struct {
Name string
Type string
SendWarning bool
SendCritical bool
Notifierr NotificationDispatcher
}
type EmailNotifier struct {
To string
type RootNotifier struct {
NotifierBase
log log.Logger
}
func (this *EmailNotifier) Dispatch(alertResult *AlertResult) {
this.log.Info("Sending email")
grafanaUrl := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
if setting.AppSubUrl != "" {
grafanaUrl += "/" + setting.AppSubUrl
func NewRootNotifier() *RootNotifier {
return &RootNotifier{
log: log.New("alerting.notifier"),
}
}
func (n *RootNotifier) Notify(context *AlertResultContext) {
n.log.Info("Sending notifications for", "ruleId", context.Rule.Id)
notifiers, err := n.getNotifiers(context.Rule.OrgId, context.Rule.Notifications)
if err != nil {
n.log.Error("Failed to read notifications", "error", err)
return
}
query := &m.GetDashboardsQuery{
DashboardIds: []int64{alertResult.AlertJob.Rule.DashboardId},
for _, notifier := range notifiers {
n.log.Info("Sending notification", "firing", context.Firing, "type", notifier.GetType())
go notifier.Notify(context)
}
}
func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Notifier, error) {
query := &m.GetAlertNotificationsQuery{OrgId: orgId, Ids: notificationIds}
if err := bus.Dispatch(query); err != nil {
return nil, err
}
var result []Notifier
for _, notification := range query.Result {
if not, err := NewNotificationFromDBModel(notification); err != nil {
return nil, err
} else {
result = append(result, not)
}
}
return result, nil
}
type NotifierBase struct {
Name string
Type string
}
func (n *NotifierBase) GetType() string {
return n.Type
}
type EmailNotifier struct {
NotifierBase
Addresses []string
log log.Logger
}
func (this *EmailNotifier) Notify(context *AlertResultContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
slugQuery := &m.GetDashboardSlugByIdQuery{Id: context.Rule.DashboardId}
if err := bus.Dispatch(slugQuery); err != nil {
this.log.Error("Failed to load dashboard", "error", err)
return
}
if len(query.Result) != 1 {
this.log.Error("Can only support one dashboard", "result", len(query.Result))
return
}
ruleLink := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slugQuery.Result, context.Rule.PanelId)
dashboard := query.Result[0]
panelId := strconv.Itoa(int(alertResult.AlertJob.Rule.PanelId))
//TODO: get from alertrule and transforms to seconds
from := "1466169458375"
to := "1466171258375"
renderUrl := fmt.Sprintf("%s/render/dashboard-solo/db/%s?from=%s&to=%s&panelId=%s&width=1000&height=500", grafanaUrl, dashboard.Slug, from, to, panelId)
cmd := &m.SendEmailCommand{
Data: map[string]interface{}{
"Name": "Name",
"State": alertResult.State,
"Description": alertResult.Description,
"TriggeredAlerts": alertResult.TriggeredAlerts,
"DashboardLink": grafanaUrl + "/dashboard/db/" + dashboard.Slug,
"AlertPageUrl": grafanaUrl + "/alerting",
"DashboardImage": renderUrl,
"RuleState": context.Rule.State,
"RuleName": context.Rule.Name,
"Severity": context.Rule.Severity,
"RuleLink": ruleLink,
},
To: []string{this.To},
To: this.Addresses,
Template: "alert_notification.html",
}
err := bus.Dispatch(cmd)
if err != nil {
this.log.Error("Could not send alert notification as email", "error", err)
this.log.Error("Failed to send alert notification email", "error", err)
}
}
type WebhookNotifier struct {
Url string
User string
Password string
log log.Logger
}
// type WebhookNotifier struct {
// Url string
// User string
// Password string
// log log.Logger
// }
//
// func (this *WebhookNotifier) Dispatch(context *AlertResultContext) {
// this.log.Info("Sending webhook")
//
// bodyJSON := simplejson.New()
// bodyJSON.Set("name", context.AlertJob.Rule.Name)
// bodyJSON.Set("state", context.State)
// bodyJSON.Set("trigged", context.TriggeredAlerts)
//
// body, _ := bodyJSON.MarshalJSON()
//
// cmd := &m.SendWebhook{
// Url: this.Url,
// User: this.User,
// Password: this.Password,
// Body: string(body),
// }
//
// bus.Dispatch(cmd)
// }
func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) {
this.log.Info("Sending webhook")
func NewNotificationFromDBModel(model *m.AlertNotification) (Notifier, error) {
if model.Type == "email" {
addressesString := model.Settings.Get("addresses").MustString()
bodyJSON := simplejson.New()
bodyJSON.Set("name", alertResult.AlertJob.Rule.Name)
bodyJSON.Set("state", alertResult.State)
bodyJSON.Set("trigged", alertResult.TriggeredAlerts)
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhook{
Url: this.Url,
User: this.User,
Password: this.Password,
Body: string(body),
}
bus.Dispatch(cmd)
}
type NotificationDispatcher interface {
Dispatch(alertResult *AlertResult)
}
func buildGetNotifiers(log log.Logger) func(orgId int64, notificationGroups []int64) []*Notification {
return func(orgId int64, notificationGroups []int64) []*Notification {
query := &m.GetAlertNotificationQuery{
OrgID: orgId,
Ids: notificationGroups,
IncludeAlwaysExecute: true,
}
err := bus.Dispatch(query)
if err != nil {
log.Error("Failed to read notifications", "error", err)
}
var result []*Notification
for _, notification := range query.Result {
not, err := NewNotificationFromDBModel(notification)
if err == nil {
result = append(result, not)
} else {
log.Error("Failed to read notification model", "error", err)
}
}
return result
}
}
func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, error) {
notifier, err := createNotifier(model.Type, model.Settings)
if err != nil {
return nil, err
}
return &Notification{
Name: model.Name,
Type: model.Type,
Notifierr: notifier,
SendCritical: model.Settings.Get("sendCrit").MustBool(),
SendWarning: model.Settings.Get("sendWarn").MustBool(),
}, nil
}
var createNotifier = func(notificationType string, settings *simplejson.Json) (NotificationDispatcher, error) {
if notificationType == "email" {
to := settings.Get("to").MustString()
if to == "" {
return nil, fmt.Errorf("Could not find to propertie in settings")
if addressesString == "" {
return nil, fmt.Errorf("Could not find addresses in settings")
}
return &EmailNotifier{
To: to,
log: log.New("alerting.notification.email"),
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Addresses: strings.Split(addressesString, "\n"),
log: log.New("alerting.notification.email"),
}, nil
}
url := settings.Get("url").MustString()
if url == "" {
return nil, fmt.Errorf("Could not find url propertie in settings")
}
return nil, errors.New("Unsupported notification type")
return &WebhookNotifier{
Url: url,
User: settings.Get("user").MustString(),
Password: settings.Get("password").MustString(),
log: log.New("alerting.notification.webhook"),
}, nil
// url := settings.Get("url").MustString()
// if url == "" {
// return nil, fmt.Errorf("Could not find url propertie in settings")
// }
//
// return &WebhookNotifier{
// Url: url,
// User: settings.Get("user").MustString(),
// Password: settings.Get("password").MustString(),
// log: log.New("alerting.notification.webhook"),
// }, nil
}
pkg/services/alerting/notifier_test.go
+112 -123
View File
@@ -1,125 +1,114 @@
package alerting
import (
"testing"
"reflect"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertNotificationExtraction(t *testing.T) {
Convey("Notifier tests", t, func() {
Convey("rules for sending notifications", func() {
dummieNotifier := NotifierImpl{}
result := &AlertResult{
State: alertstates.Critical,
}
notifier := &Notification{
Name: "Test Notifier",
Type: "TestType",
SendCritical: true,
SendWarning: true,
}
Convey("Should send notification", func() {
So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeTrue)
})
Convey("warn:false and state:warn should not send", func() {
result.State = alertstates.Warn
notifier.SendWarning = false
So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeFalse)
})
})
Convey("Parsing alert notification from settings", func() {
Convey("Parsing email", func() {
Convey("empty settings should return error", func() {
json := `{ }`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
_, err := NewNotificationFromDBModel(model)
So(err, ShouldNotBeNil)
})
Convey("from settings", func() {
json := `
{
"to": "ops@grafana.org"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
not, err := NewNotificationFromDBModel(model)
So(err, ShouldBeNil)
So(not.Name, ShouldEqual, "ops")
So(not.Type, ShouldEqual, "email")
So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier")
email := not.Notifierr.(*EmailNotifier)
So(email.To, ShouldEqual, "ops@grafana.org")
})
})
Convey("Parsing webhook", func() {
Convey("empty settings should return error", func() {
json := `{ }`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "webhook",
Settings: settingsJSON,
}
_, err := NewNotificationFromDBModel(model)
So(err, ShouldNotBeNil)
})
Convey("from settings", func() {
json := `
{
"url": "http://localhost:3000",
"username": "username",
"password": "password"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "slack",
Type: "webhook",
Settings: settingsJSON,
}
not, err := NewNotificationFromDBModel(model)
So(err, ShouldBeNil)
So(not.Name, ShouldEqual, "slack")
So(not.Type, ShouldEqual, "webhook")
So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier")
webhook := not.Notifierr.(*WebhookNotifier)
So(webhook.Url, ShouldEqual, "http://localhost:3000")
})
})
})
})
}
// func TestAlertNotificationExtraction(t *testing.T) {
// Convey("Notifier tests", t, func() {
// Convey("rules for sending notifications", func() {
// dummieNotifier := NotifierImpl{}
//
// result := &AlertResult{
// State: alertstates.Critical,
// }
//
// notifier := &Notification{
// Name: "Test Notifier",
// Type: "TestType",
// SendCritical: true,
// SendWarning: true,
// }
//
// Convey("Should send notification", func() {
// So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeTrue)
// })
//
// Convey("warn:false and state:warn should not send", func() {
// result.State = alertstates.Warn
// notifier.SendWarning = false
// So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeFalse)
// })
// })
//
// Convey("Parsing alert notification from settings", func() {
// Convey("Parsing email", func() {
// Convey("empty settings should return error", func() {
// json := `{ }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "email",
// Settings: settingsJSON,
// }
//
// _, err := NewNotificationFromDBModel(model)
// So(err, ShouldNotBeNil)
// })
//
// Convey("from settings", func() {
// json := `
// {
// "to": "ops@grafana.org"
// }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "email",
// Settings: settingsJSON,
// }
//
// not, err := NewNotificationFromDBModel(model)
//
// So(err, ShouldBeNil)
// So(not.Name, ShouldEqual, "ops")
// So(not.Type, ShouldEqual, "email")
// So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier")
//
// email := not.Notifierr.(*EmailNotifier)
// So(email.To, ShouldEqual, "ops@grafana.org")
// })
// })
//
// Convey("Parsing webhook", func() {
// Convey("empty settings should return error", func() {
// json := `{ }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "webhook",
// Settings: settingsJSON,
// }
//
// _, err := NewNotificationFromDBModel(model)
// So(err, ShouldNotBeNil)
// })
//
// Convey("from settings", func() {
// json := `
// {
// "url": "http://localhost:3000",
// "username": "username",
// "password": "password"
// }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "slack",
// Type: "webhook",
// Settings: settingsJSON,
// }
//
// not, err := NewNotificationFromDBModel(model)
//
// So(err, ShouldBeNil)
// So(not.Name, ShouldEqual, "slack")
// So(not.Type, ShouldEqual, "webhook")
// So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier")
//
// webhook := not.Notifierr.(*WebhookNotifier)
// So(webhook.Url, ShouldEqual, "http://localhost:3000")
// })
// })
// })
// })
// }
pkg/services/alerting/reader.go
+13 -8
View File
@@ -18,10 +18,13 @@ type AlertRuleReader struct {
serverID string
serverPosition int
clusterSize int
log log.Logger
}
func NewRuleReader() *AlertRuleReader {
ruleReader := &AlertRuleReader{}
ruleReader := &AlertRuleReader{
log: log.New("alerting.ruleReader"),
}
go ruleReader.initReader()
return ruleReader
@@ -40,17 +43,19 @@ func (arr *AlertRuleReader) initReader() {
func (arr *AlertRuleReader) Fetch() []*AlertRule {
cmd := &m.GetAllAlertsQuery{}
err := bus.Dispatch(cmd)
if err != nil {
log.Error(1, "Alerting: ruleReader.fetch(): Could not load alerts", err)
if err := bus.Dispatch(cmd); err != nil {
arr.log.Error("Could not load alerts", "error", err)
return []*AlertRule{}
}
res := make([]*AlertRule, len(cmd.Result))
for i, ruleDef := range cmd.Result {
model, _ := NewAlertRuleFromDBModel(ruleDef)
res[i] = model
res := make([]*AlertRule, 0)
for _, ruleDef := range cmd.Result {
if model, err := NewAlertRuleFromDBModel(ruleDef); err != nil {
arr.log.Error("Could not build alert model for rule", "ruleId", ruleDef.Id, "error", err)
} else {
res = append(res, model)
}
}
return res
pkg/services/alerting/result_handler.go
+23 -38
View File
@@ -1,16 +1,13 @@
package alerting
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
)
type ResultHandler interface {
Handle(result *AlertResult)
Handle(result *AlertResultContext)
}
type ResultHandlerImpl struct {
@@ -20,49 +17,37 @@ type ResultHandlerImpl struct {
func NewResultHandler() *ResultHandlerImpl {
return &ResultHandlerImpl{
log: log.New("alerting.responseHandler"),
notifier: NewNotifier(),
log: log.New("alerting.resultHandler"),
notifier: NewRootNotifier(),
}
}
func (handler *ResultHandlerImpl) Handle(result *AlertResult) {
if handler.shouldUpdateState(result) {
cmd := &m.UpdateAlertStateCommand{
AlertId: result.AlertJob.Rule.Id,
State: result.State,
Info: result.Description,
OrgId: result.AlertJob.Rule.OrgId,
TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts),
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
var newState m.AlertStateType
if result.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
newState = m.AlertStatePending
} else if result.Firing {
newState = m.AlertStateFiring
} else {
newState = m.AlertStateOK
}
if result.Rule.State != newState {
handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
cmd := &m.SetAlertStateCommand{
AlertId: result.Rule.Id,
OrgId: result.Rule.OrgId,
State: newState,
}
if err := bus.Dispatch(cmd); err != nil {
handler.log.Error("Failed to save state", "error", err)
}
handler.log.Debug("will notify about new state", "new state", result.State)
result.Rule.State = newState
handler.notifier.Notify(result)
}
}
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool {
query := &m.GetLastAlertStateQuery{
AlertId: result.AlertJob.Rule.Id,
OrgId: result.AlertJob.Rule.OrgId,
}
if err := bus.Dispatch(query); err != nil {
log.Error2("Failed to read last alert state", "error", err)
return false
}
if query.Result == nil {
return true
}
lastExecution := query.Result.Created
asdf := result.StartTime.Add(time.Minute * -15)
olderThen15Min := lastExecution.Before(asdf)
changedState := query.Result.State != result.State
return changedState || olderThen15Min
}
pkg/services/alerting/result_handler_test.go
+56 -57
View File
@@ -1,59 +1,58 @@
package alerting
import (
"testing"
"time"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertResultHandler(t *testing.T) {
Convey("Test result Handler", t, func() {
resultHandler := ResultHandlerImpl{}
mockResult := &AlertResult{
State: alertstates.Ok,
AlertJob: &AlertJob{
Rule: &AlertRule{
Id: 1,
OrgId: 1,
},
},
}
mockAlertState := &m.AlertState{}
bus.ClearBusHandlers()
bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error {
query.Result = mockAlertState
return nil
})
Convey("Should update", func() {
Convey("when no earlier alert state", func() {
mockAlertState = nil
So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
})
Convey("alert state have changed", func() {
mockAlertState = &m.AlertState{
NewState: alertstates.Critical,
}
mockResult.State = alertstates.Ok
So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
})
Convey("last alert state was 15min ago", func() {
now := time.Now()
mockAlertState = &m.AlertState{
NewState: alertstates.Critical,
Created: now.Add(time.Minute * -30),
}
mockResult.State = alertstates.Critical
mockResult.ExeuctionTime = time.Now()
So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
})
})
})
}
// import (
// "testing"
// "time"
//
// "github.com/grafana/grafana/pkg/bus"
// m "github.com/grafana/grafana/pkg/models"
// "github.com/grafana/grafana/pkg/services/alerting/alertstates"
//
// . "github.com/smartystreets/goconvey/convey"
// )
//
// func TestAlertResultHandler(t *testing.T) {
// Convey("Test result Handler", t, func() {
// resultHandler := ResultHandlerImpl{}
// mockResult := &AlertResultContext{
// Triggered: false,
// Rule: &AlertRule{
// Id: 1,
// OrgId 1,
// },
// }
// mockAlertState := &m.AlertState{}
// bus.ClearBusHandlers()
// bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error {
// query.Result = mockAlertState
// return nil
// })
//
// Convey("Should update", func() {
//
// Convey("when no earlier alert state", func() {
// mockAlertState = nil
// So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
// })
//
// Convey("alert state have changed", func() {
// mockAlertState = &m.AlertState{
// State: alertstates.Critical,
// }
// mockResult.Triggered = false
// So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
// })
//
// Convey("last alert state was 15min ago", func() {
// now := time.Now()
// mockAlertState = &m.AlertState{
// State: alertstates.Critical,
// Created: now.Add(time.Minute * -30),
// }
// mockResult.Triggered = true
// mockResult.StartTime = time.Now()
// So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
// })
// })
// })
// }
pkg/services/alerting/test_rule.go
+57
View File
@@ -0,0 +1,57 @@
package alerting
import (
"fmt"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
)
type AlertTestCommand struct {
Dashboard *simplejson.Json
PanelId int64
OrgId int64
Result *AlertResultContext
}
func init() {
bus.AddHandler("alerting", handleAlertTestCommand)
}
func handleAlertTestCommand(cmd *AlertTestCommand) error {
dash := m.NewDashboardFromJson(cmd.Dashboard)
extractor := NewDashAlertExtractor(dash, cmd.OrgId)
alerts, err := extractor.GetAlerts()
if err != nil {
return err
}
for _, alert := range alerts {
if alert.PanelId == cmd.PanelId {
rule, err := NewAlertRuleFromDBModel(alert)
if err != nil {
return err
}
cmd.Result = testAlertRule(rule)
return nil
}
}
return fmt.Errorf("Could not find alert with panel id %d", cmd.PanelId)
}
func testAlertRule(rule *AlertRule) *AlertResultContext {
handler := NewHandler()
context := NewAlertResultContext(rule)
context.IsTestRun = true
handler.Execute(context)
return context
}
pkg/services/alerting/transformers/aggregation.go
-71
View File
@@ -1,71 +0,0 @@
package transformers
import (
"fmt"
"math"
"github.com/grafana/grafana/pkg/tsdb"
)
func NewAggregationTransformer(method string) *AggregationTransformer {
return &AggregationTransformer{
Method: method,
}
}
type AggregationTransformer struct {
Method string
}
func (at *AggregationTransformer) Transform(timeserie *tsdb.TimeSeries) (float64, error) {
if at.Method == "avg" {
sum := float64(0)
for _, point := range timeserie.Points {
sum += point[0]
}
return sum / float64(len(timeserie.Points)), nil
}
if at.Method == "sum" {
sum := float64(0)
for _, v := range timeserie.Points {
sum += v[0]
}
return sum, nil
}
if at.Method == "min" {
min := timeserie.Points[0][0]
for _, v := range timeserie.Points {
if v[0] < min {
min = v[0]
}
}
return min, nil
}
if at.Method == "max" {
max := timeserie.Points[0][0]
for _, v := range timeserie.Points {
if v[0] > max {
max = v[0]
}
}
return max, nil
}
if at.Method == "mean" {
midPosition := int64(math.Floor(float64(len(timeserie.Points)) / float64(2)))
return timeserie.Points[midPosition][0], nil
}
return float64(0), fmt.Errorf("Missing method")
}
pkg/services/alerting/transformers/transformer.go
-7
View File
@@ -1,7 +0,0 @@
package transformers
import "github.com/grafana/grafana/pkg/tsdb"
type Transformer interface {
Transform(timeserie *tsdb.TimeSeries) (float64, error)
}
pkg/services/sqlstore/alert.go
+19 -45
View File
@@ -17,52 +17,9 @@ func init() {
bus.AddHandler("sql", GetAlertById)
bus.AddHandler("sql", DeleteAlertById)
bus.AddHandler("sql", GetAllAlertQueryHandler)
//bus.AddHandler("sql", HeartBeat)
bus.AddHandler("sql", SetAlertState)
}
/*
func HeartBeat(query *m.HeartBeatCommand) error {
return inTransaction(func(sess *xorm.Session) error {
now := time.Now().Sub(0, 0, 0, 5)
activeTime := time.Now().Sub(0, 0, 0, 5)
ownHeartbeats := make([]m.HeartBeat, 0)
err := x.Where("server_id = ?", query.ServerId).Find(&ownHeartbeats)
if err != nil {
return err
}
if (len(ownHeartbeats)) > 0 && ownHeartbeats[0].Updated > activeTime {
//update
x.Insert(&m.HeartBeat{ServerId: query.ServerId, Created: now, Updated: now})
} else {
thisServer := ownHeartbeats[0]
thisServer.Updated = now
x.Id(thisServer.Id).Update(&thisServer)
}
activeServers := make([]m.HeartBeat, 0)
err = x.Where("server_id = ? and updated > ", query.ServerId, now.String()).OrderBy("id").Find(&activeServers)
if err != nil {
return err
}
for i, pos := range activeServers {
if pos.ServerId == query.ServerId {
query.Result = &m.AlertingClusterInfo{
ClusterSize: len(activeServers),
UptimePosition: i,
}
return nil
}
}
return nil
})
}
*/
func GetAlertById(query *m.GetAlertByIdQuery) error {
alert := m.Alert{}
has, err := x.Id(query.Id).Get(&alert)
@@ -203,7 +160,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
} else {
alert.Updated = time.Now()
alert.Created = time.Now()
alert.State = "UNKNOWN"
alert.State = m.AlertStatePending
alert.CreatedBy = cmd.UserId
alert.UpdatedBy = cmd.UserId
@@ -253,3 +210,20 @@ func GetAlertsByDashboardId2(dashboardId int64, sess *xorm.Session) ([]*m.Alert,
return alerts, nil
}
func SetAlertState(cmd *m.SetAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error {
alert := m.Alert{}
if has, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
return err
} else if !has {
return fmt.Errorf("Could not find alert")
}
alert.State = cmd.State
sess.Id(alert.Id).Update(&alert)
return nil
})
}
pkg/services/sqlstore/alert_notification.go
+40 -80
View File
@@ -3,7 +3,7 @@ package sqlstore
import (
"bytes"
"fmt"
"strconv"
"strings"
"time"
"github.com/go-xorm/xorm"
@@ -31,11 +31,11 @@ func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
})
}
func AlertNotificationQuery(query *m.GetAlertNotificationQuery) error {
func AlertNotificationQuery(query *m.GetAlertNotificationsQuery) error {
return getAlertNotifications(query, x.NewSession())
}
func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Session) error {
func getAlertNotifications(query *m.GetAlertNotificationsQuery, sess *xorm.Session) error {
var sql bytes.Buffer
params := make([]interface{}, 0)
@@ -43,16 +43,15 @@ func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Sessio
alert_notification.id,
alert_notification.org_id,
alert_notification.name,
alert_notification.type,
alert_notification.type,
alert_notification.created,
alert_notification.updated,
alert_notification.settings,
alert_notification.always_execute
alert_notification.updated,
alert_notification.settings
FROM alert_notification
`)
sql.WriteString(` WHERE alert_notification.org_id = ?`)
params = append(params, query.OrgID)
params = append(params, query.OrgId)
if query.Name != "" {
sql.WriteString(` AND alert_notification.name = ?`)
@@ -61,60 +60,28 @@ func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Sessio
if query.Id != 0 {
sql.WriteString(` AND alert_notification.id = ?`)
params = append(params, strconv.Itoa(int(query.Id)))
params = append(params, query.Id)
}
if len(query.Ids) > 0 {
sql.WriteString(` AND (`)
for i, id := range query.Ids {
if i != 0 {
sql.WriteString(` OR`)
}
sql.WriteString(` alert_notification.id = ?`)
params = append(params, id)
sql.WriteString(` AND alert_notification.id IN (?` + strings.Repeat(",?", len(query.Ids)-1) + ")")
for _, v := range query.Ids {
params = append(params, v)
}
sql.WriteString(`)`)
}
var searches []*m.AlertNotification
if err := sess.Sql(sql.String(), params...).Find(&searches); err != nil {
results := make([]*m.AlertNotification, 0)
if err := sess.Sql(sql.String(), params...).Find(&results); err != nil {
return err
}
var result []*m.AlertNotification
var def []*m.AlertNotification
if query.IncludeAlwaysExecute {
if err := sess.Where("org_id = ? AND always_execute = 1", query.OrgID).Find(&def); err != nil {
return err
}
result = append(result, def...)
}
for _, s := range searches {
canAppend := true
for _, d := range result {
if d.Id == s.Id {
canAppend = false
break
}
}
if canAppend {
result = append(result, s)
}
}
query.Result = result
query.Result = results
return nil
}
func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error {
return inTransaction(func(sess *xorm.Session) error {
existingQuery := &m.GetAlertNotificationQuery{OrgID: cmd.OrgID, Name: cmd.Name, IncludeAlwaysExecute: false}
existingQuery := &m.GetAlertNotificationsQuery{OrgId: cmd.OrgId, Name: cmd.Name}
err := getAlertNotifications(existingQuery, sess)
if err != nil {
@@ -126,18 +93,15 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
}
alertNotification := &m.AlertNotification{
OrgId: cmd.OrgID,
Name: cmd.Name,
Type: cmd.Type,
Created: time.Now(),
Settings: cmd.Settings,
Updated: time.Now(),
AlwaysExecute: cmd.AlwaysExecute,
OrgId: cmd.OrgId,
Name: cmd.Name,
Type: cmd.Type,
Settings: cmd.Settings,
Created: time.Now(),
Updated: time.Now(),
}
_, err = sess.Insert(alertNotification)
if err != nil {
if _, err = sess.Insert(alertNotification); err != nil {
return err
}
@@ -148,38 +112,34 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
return inTransaction(func(sess *xorm.Session) (err error) {
current := &m.AlertNotification{}
_, err = sess.Id(cmd.Id).Get(current)
current := m.AlertNotification{}
if err != nil {
if _, err = sess.Id(cmd.Id).Get(&current); err != nil {
return err
}
alertNotification := &m.AlertNotification{
Id: cmd.Id,
OrgId: cmd.OrgID,
Name: cmd.Name,
Type: cmd.Type,
Settings: cmd.Settings,
Updated: time.Now(),
Created: current.Created,
AlwaysExecute: cmd.AlwaysExecute,
}
sess.UseBool("always_execute")
var affected int64
affected, err = sess.Id(alertNotification.Id).Update(alertNotification)
if err != nil {
// check if name exists
sameNameQuery := &m.GetAlertNotificationsQuery{OrgId: cmd.OrgId, Name: cmd.Name}
if err := getAlertNotifications(sameNameQuery, sess); err != nil {
return err
}
if affected == 0 {
if len(sameNameQuery.Result) > 0 && sameNameQuery.Result[0].Id != current.Id {
return fmt.Errorf("Alert notification name %s already exists", cmd.Name)
}
current.Updated = time.Now()
current.Settings = cmd.Settings
current.Name = cmd.Name
current.Type = cmd.Type
if affected, err := sess.Id(cmd.Id).Update(current); err != nil {
return err
} else if affected == 0 {
return fmt.Errorf("Could not find alert notification")
}
cmd.Result = alertNotification
cmd.Result = &current
return nil
})
}
pkg/services/sqlstore/alert_state.go
+75 -76
View File
@@ -1,78 +1,77 @@
package sqlstore
import (
"fmt"
"time"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
)
func init() {
bus.AddHandler("sql", SetNewAlertState)
bus.AddHandler("sql", GetAlertStateLogByAlertId)
bus.AddHandler("sql", GetLastAlertStateQuery)
}
func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
states := make([]m.AlertState, 0)
if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
return err
}
if len(states) == 0 {
cmd.Result = nil
return nil
}
cmd.Result = &states[0]
return nil
}
func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
return inTransaction(func(sess *xorm.Session) error {
if !cmd.IsValidState() {
return fmt.Errorf("new state is invalid")
}
alert := m.Alert{}
has, err := sess.Id(cmd.AlertId).Get(&alert)
if err != nil {
return err
}
if !has {
return fmt.Errorf("Could not find alert")
}
alert.State = cmd.State
sess.Id(alert.Id).Update(&alert)
alertState := m.AlertState{
AlertId: cmd.AlertId,
OrgId: cmd.OrgId,
State: cmd.State,
Info: cmd.Info,
Created: time.Now(),
TriggeredAlerts: cmd.TriggeredAlerts,
}
sess.Insert(&alertState)
cmd.Result = &alert
return nil
})
}
func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
states := make([]m.AlertState, 0)
if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
return err
}
cmd.Result = &states
return nil
}
// import (
// "fmt"
// "time"
//
// "github.com/go-xorm/xorm"
// "github.com/grafana/grafana/pkg/bus"
// m "github.com/grafana/grafana/pkg/models"
// )
//
// func init() {
// bus.AddHandler("sql", SetNewAlertState)
// bus.AddHandler("sql", GetAlertStateLogByAlertId)
// bus.AddHandler("sql", GetLastAlertStateQuery)
// }
//
// func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
// states := make([]m.AlertState, 0)
//
// if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
// return err
// }
//
// if len(states) == 0 {
// cmd.Result = nil
// return nil
// }
//
// cmd.Result = &states[0]
// return nil
// }
//
// func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
// return inTransaction(func(sess *xorm.Session) error {
// if !cmd.IsValidState() {
// return fmt.Errorf("new state is invalid")
// }
//
// alert := m.Alert{}
// has, err := sess.Id(cmd.AlertId).Get(&alert)
// if err != nil {
// return err
// }
//
// if !has {
// return fmt.Errorf("Could not find alert")
// }
//
// alert.State = cmd.State
// sess.Id(alert.Id).Update(&alert)
//
// alertState := m.AlertState{
// AlertId: cmd.AlertId,
// OrgId: cmd.OrgId,
// State: cmd.State,
// Info: cmd.Info,
// Created: time.Now(),
// }
//
// sess.Insert(&alertState)
//
// cmd.Result = &alert
// return nil
// })
// }
//
// func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
// states := make([]m.AlertState, 0)
//
// if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
// return err
// }
//
// cmd.Result = &states
// return nil
// }
pkg/services/sqlstore/alert_state_test.go
+98 -98
View File
@@ -1,100 +1,100 @@
package sqlstore
import (
"testing"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertingStateAccess(t *testing.T) {
Convey("Test alerting state changes", t, func() {
InitTestDB(t)
testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
items := []*m.Alert{
{
PanelId: 1,
DashboardId: testDash.Id,
OrgId: testDash.OrgId,
Name: "Alerting title",
Description: "Alerting description",
},
}
cmd := m.SaveAlertsCommand{
Alerts: items,
DashboardId: testDash.Id,
OrgId: 1,
UserId: 1,
}
err := SaveAlerts(&cmd)
So(err, ShouldBeNil)
Convey("Cannot insert invalid states", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "maybe ok",
Info: "Shit just hit the fan",
})
So(err, ShouldNotBeNil)
})
Convey("Changes state to alert", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "CRITICAL",
Info: "Shit just hit the fan",
})
Convey("can get new state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query)
So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "CRITICAL")
})
Convey("Changes state to ok", func() {
err = SetNewAlertState(&m.UpdateAlertStateCommand{
AlertId: 1,
NewState: "OK",
Info: "Shit just hit the fan",
})
Convey("get ok state for alert", func() {
query := &m.GetAlertByIdQuery{Id: 1}
err := GetAlertById(query)
So(err, ShouldBeNil)
So(query.Result.State, ShouldEqual, "OK")
})
Convey("should have two event state logs", func() {
query := &m.GetAlertsStateQuery{
AlertId: 1,
OrgId: 1,
}
err := GetAlertStateLogByAlertId(query)
So(err, ShouldBeNil)
So(len(*query.Result), ShouldEqual, 2)
})
Convey("should not get any alerts with critical state", func() {
query := &m.GetAlertsQuery{
OrgId: 1,
State: []string{"Critical", "Warn"},
}
err := HandleAlertsQuery(query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
})
})
})
}
// import (
// "testing"
//
// m "github.com/grafana/grafana/pkg/models"
// . "github.com/smartystreets/goconvey/convey"
// )
//
// func TestAlertingStateAccess(t *testing.T) {
// Convey("Test alerting state changes", t, func() {
// InitTestDB(t)
//
// testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
//
// items := []*m.Alert{
// {
// PanelId: 1,
// DashboardId: testDash.Id,
// OrgId: testDash.OrgId,
// Name: "Alerting title",
// Description: "Alerting description",
// },
// }
//
// cmd := m.SaveAlertsCommand{
// Alerts: items,
// DashboardId: testDash.Id,
// OrgId: 1,
// UserId: 1,
// }
//
// err := SaveAlerts(&cmd)
// So(err, ShouldBeNil)
//
// Convey("Cannot insert invalid states", func() {
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "maybe ok",
// Info: "Shit just hit the fan",
// })
//
// So(err, ShouldNotBeNil)
// })
//
// Convey("Changes state to alert", func() {
//
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "CRITICAL",
// Info: "Shit just hit the fan",
// })
//
// Convey("can get new state for alert", func() {
// query := &m.GetAlertByIdQuery{Id: 1}
// err := GetAlertById(query)
// So(err, ShouldBeNil)
// So(query.Result.State, ShouldEqual, "CRITICAL")
// })
//
// Convey("Changes state to ok", func() {
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
// AlertId: 1,
// NewState: "OK",
// Info: "Shit just hit the fan",
// })
//
// Convey("get ok state for alert", func() {
// query := &m.GetAlertByIdQuery{Id: 1}
// err := GetAlertById(query)
// So(err, ShouldBeNil)
// So(query.Result.State, ShouldEqual, "OK")
// })
//
// Convey("should have two event state logs", func() {
// query := &m.GetAlertsStateQuery{
// AlertId: 1,
// OrgId: 1,
// }
//
// err := GetAlertStateLogByAlertId(query)
// So(err, ShouldBeNil)
//
// So(len(*query.Result), ShouldEqual, 2)
// })
//
// Convey("should not get any alerts with critical state", func() {
// query := &m.GetAlertsQuery{
// OrgId: 1,
// State: []string{"Critical", "Warn"},
// }
//
// err := HandleAlertsQuery(query)
// So(err, ShouldBeNil)
// So(len(query.Result), ShouldEqual, 0)
// })
// })
// })
// })
// }
pkg/services/sqlstore/migrations/alert_mig.go
+1 -1
View File
@@ -19,6 +19,7 @@ func addAlertMigrations(mg *Migrator) {
{Name: "settings", Type: DB_Text, Nullable: false},
{Name: "frequency", Type: DB_BigInt, Nullable: false},
{Name: "handler", Type: DB_BigInt, Nullable: false},
{Name: "severity", Type: DB_Text, Nullable: false},
{Name: "enabled", Type: DB_Bool, Nullable: false},
{Name: "created", Type: DB_DateTime, Nullable: false},
{Name: "updated", Type: DB_DateTime, Nullable: false},
@@ -64,7 +65,6 @@ func addAlertMigrations(mg *Migrator) {
{Name: "org_id", Type: DB_BigInt, Nullable: false},
{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
{Name: "always_execute", Type: DB_Bool, Nullable: false},
{Name: "settings", Type: DB_Text, Nullable: false},
{Name: "created", Type: DB_DateTime, Nullable: false},
{Name: "updated", Type: DB_DateTime, Nullable: false},
pkg/tsdb/graphite/graphite.go
+7 -5
View File
@@ -31,16 +31,15 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
result := &tsdb.BatchResult{}
params := url.Values{
"from": []string{formatTimeRange(context.TimeRange.From)},
"until": []string{context.TimeRange.To},
"from": []string{"-" + formatTimeRange(context.TimeRange.From)},
"until": []string{formatTimeRange(context.TimeRange.To)},
"format": []string{"json"},
"maxDataPoints": []string{"500"},
}
for _, query := range queries {
params["target"] = []string{
query.Query,
}
params["target"] = []string{query.Query}
glog.Debug("Graphite request", "query", query.Query)
}
client := http.Client{Timeout: time.Duration(10 * time.Second)}
@@ -79,5 +78,8 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
}
func formatTimeRange(input string) string {
if input == "now" {
return input
}
return strings.Replace(strings.Replace(input, "m", "min", -1), "M", "mon", -1)
}
pkg/tsdb/models.go
+2 -2
View File
@@ -46,8 +46,8 @@ type QueryResult struct {
}
type TimeSeries struct {
Name string
Points [][2]float64
Name string `json:"name"`
Points [][2]float64 `json:"points"`
}
type TimeSeriesSlice []*TimeSeries
pkg/tsdb/request.go
+2
View File
@@ -1,5 +1,7 @@
package tsdb
type HandleRequestFunc func(req *Request) (*Response, error)
func HandleRequest(req *Request) (*Response, error) {
context := NewQueryContext(req.Queries, req.TimeRange)