Merge branch 'alerting_notifications' into alerting

pkg/api/alerting.go

@@ -82,7 +82,7 @@ func GetAlerts(c *middleware.Context) Response {
 
 	//TODO: should be possible to speed this up with lookup table
 	for _, alert := range alertDTOs {
-		for _, dash := range *dashboardsQuery.Result {
+		for _, dash := range dashboardsQuery.Result {
 			if alert.DashboardId == dash.Id {
 				alert.DashbboardUri = "db/" + dash.Slug
 			}
@@ -140,6 +140,7 @@ func GetAlertStates(c *middleware.Context) Response {
 // PUT /api/alerts/events/:id
 func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
 	cmd.AlertId = c.ParamsInt64(":alertId")
+	cmd.OrgId = c.OrgId
 
 	query := models.GetAlertByIdQuery{Id: cmd.AlertId}
 	if err := bus.Dispatch(&query); err != nil {
@@ -156,3 +157,73 @@ func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Re
 
 	return Json(200, cmd.Result)
 }
+
+func GetAlertNotifications(c *middleware.Context) Response {
+	query := &models.GetAlertNotificationQuery{
+		OrgID: c.OrgId,
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		return ApiError(500, "Failed to get alert notifications", err)
+	}
+
+	var result []dtos.AlertNotificationDTO
+
+	for _, notification := range query.Result {
+		result = append(result, dtos.AlertNotificationDTO{
+			Id:      notification.Id,
+			Name:    notification.Name,
+			Type:    notification.Type,
+			Created: notification.Created,
+			Updated: notification.Updated,
+		})
+	}
+
+	return Json(200, result)
+}
+
+func GetAlertNotificationById(c *middleware.Context) Response {
+	query := &models.GetAlertNotificationQuery{
+		OrgID: c.OrgId,
+		Id:    c.ParamsInt64("notificationId"),
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		return ApiError(500, "Failed to get alert notifications", err)
+	}
+
+	return Json(200, query.Result[0])
+}
+
+func CreateAlertNotification(c *middleware.Context, cmd models.CreateAlertNotificationCommand) Response {
+	cmd.OrgID = c.OrgId
+
+	if err := bus.Dispatch(&cmd); err != nil {
+		return ApiError(500, "Failed to create alert notification", err)
+	}
+
+	return Json(200, cmd.Result)
+}
+
+func UpdateAlertNotification(c *middleware.Context, cmd models.UpdateAlertNotificationCommand) Response {
+	cmd.OrgID = c.OrgId
+
+	if err := bus.Dispatch(&cmd); err != nil {
+		return ApiError(500, "Failed to update alert notification", err)
+	}
+
+	return Json(200, cmd.Result)
+}
+
+func DeleteAlertNotification(c *middleware.Context) Response {
+	cmd := models.DeleteAlertNotificationCommand{
+		OrgId: c.OrgId,
+		Id:    c.ParamsInt64("notificationId"),
+	}
+
+	if err := bus.Dispatch(&cmd); err != nil {
+		return ApiError(500, "Failed to delete alert notification", err)
+	}
+
+	return Json(200, map[string]interface{}{"notificationId": cmd.Id})
+}

pkg/api/api.go

@@ -62,6 +62,7 @@ func Register(r *macaron.Macaron) {
 	r.Get("/playlists/", reqSignedIn, Index)
 	r.Get("/playlists/*", reqSignedIn, Index)
 	r.Get("/alerting/", reqSignedIn, Index)
+	r.Get("/alerting/*", reqSignedIn, Index)
 
 	// sign up
 	r.Get("/signup", Index)
@@ -247,13 +248,22 @@ func Register(r *macaron.Macaron) {
 		r.Group("/alerts", func() {
 			r.Group("/rules", func() {
 				r.Get("/:alertId/states", wrap(GetAlertStates))
-				r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState))
+				//r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState))
 				r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert))
 				//r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates
 				r.Get("/", wrap(GetAlerts))
 			})
 
-			r.Get("/changes", wrap(GetAlertChanges))
+			r.Get("/notifications", wrap(GetAlertNotifications))
+
+			r.Group("/notification", func() {
+				r.Post("/", bind(m.CreateAlertNotificationCommand{}), wrap(CreateAlertNotification))
+				r.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification))
+				r.Get("/:notificationId", wrap(GetAlertNotificationById))
+				r.Delete("/:notificationId", wrap(DeleteAlertNotification))
+			}, reqOrgAdmin)
+
+			//r.Get("/changes", wrap(GetAlertChanges))
 		})
 
 		// error test

pkg/api/dtos/alerting.go

@@ -1,5 +1,7 @@
 package dtos
 
+import "time"
+
 type AlertRuleDTO struct {
 	Id           int64   `json:"id"`
 	DashboardId  int64   `json:"dashboardId"`
@@ -19,3 +21,11 @@ type AlertRuleDTO struct {
 
 	DashbboardUri string `json:"dashboardUri"`
 }
+
+type AlertNotificationDTO struct {
+	Id      int64     `json:"id"`
+	Name    string    `json:"name"`
+	Type    string    `json:"type"`
+	Created time.Time `json:"created"`
+	Updated time.Time `json:"updated"`
+}

pkg/api/index.go

@@ -80,10 +80,16 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) {
 	})
 
 	if setting.AlertingEnabled && (c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR) {
+		alertChildNavs := []*dtos.NavLink{
+			{Text: "Home", Url: setting.AppSubUrl + "/alerting"},
+			{Text: "Notifications", Url: setting.AppSubUrl + "/alerting/notifications"},
+		}
+
 		data.MainNavLinks = append(data.MainNavLinks, &dtos.NavLink{
-			Text: "Alerting",
-			Icon: "icon-gf icon-gf-monitoring",
-			Url:  setting.AppSubUrl + "/alerting",
+			Text:     "Alerting",
+			Icon:     "icon-gf icon-gf-monitoring",
+			Url:      setting.AppSubUrl + "/alerting",
+			Children: alertChildNavs,
 		})
 	}
 

pkg/api/playlist_play.go

@@ -18,7 +18,7 @@ func populateDashboardsById(dashboardByIds []int64) ([]m.PlaylistDashboardDto, e
 			return result, err
 		}
 
-		for _, item := range *dashboardQuery.Result {
+		for _, item := range dashboardQuery.Result {
 			result = append(result, m.PlaylistDashboardDto{
 				Id:    item.Id,
 				Slug:  item.Slug,

pkg/models/alert.go

@@ -28,6 +28,10 @@ func (alert *Alert) ValidToSave() bool {
 	return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0
 }
 
+func (alert *Alert) ShouldUpdateState(newState string) bool {
+	return alert.State != newState
+}
+
 func (this *Alert) ContainsUpdates(other *Alert) bool {
 	result := false
 	result = result || this.Name != other.Name

pkg/models/alert_notifications.go

@@ -0,0 +1,54 @@
+package models
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+)
+
+type AlertNotification struct {
+	Id            int64            `json:"id"`
+	OrgId         int64            `json:"-"`
+	Name          string           `json:"name"`
+	Type          string           `json:"type"`
+	AlwaysExecute bool             `json:"alwaysExecute"`
+	Settings      *simplejson.Json `json:"settings"`
+	Created       time.Time        `json:"created"`
+	Updated       time.Time        `json:"updated"`
+}
+
+type CreateAlertNotificationCommand struct {
+	Name          string           `json:"name"  binding:"Required"`
+	Type          string           `json:"type"  binding:"Required"`
+	AlwaysExecute bool             `json:"alwaysExecute"`
+	OrgID         int64            `json:"-"`
+	Settings      *simplejson.Json `json:"settings"`
+
+	Result *AlertNotification
+}
+
+type UpdateAlertNotificationCommand struct {
+	Id            int64            `json:"id"  binding:"Required"`
+	Name          string           `json:"name"  binding:"Required"`
+	Type          string           `json:"type"  binding:"Required"`
+	AlwaysExecute bool             `json:"alwaysExecute"`
+	OrgID         int64            `json:"-"`
+	Settings      *simplejson.Json `json:"settings"  binding:"Required"`
+
+	Result *AlertNotification
+}
+
+type DeleteAlertNotificationCommand struct {
+	Id    int64
+	OrgId int64
+}
+
+type GetAlertNotificationQuery struct {
+	Name                 string
+	Id                   int64
+	Ids                  []int64
+	OrgID                int64
+	IncludeAlwaysExecute bool
+
+	Result []*AlertNotification
+}

pkg/models/alert_state.go

@@ -3,16 +3,18 @@ package models
 import (
 	"time"
 
+	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
 )
 
 type AlertState struct {
-	Id       int64     `json:"-"`
-	OrgId    int64     `json:"-"`
-	AlertId  int64     `json:"alertId"`
-	NewState string    `json:"newState"`
-	Created  time.Time `json:"created"`
-	Info     string    `json:"info"`
+	Id              int64            `json:"-"`
+	OrgId           int64            `json:"-"`
+	AlertId         int64            `json:"alertId"`
+	NewState        string           `json:"newState"`
+	Created         time.Time        `json:"created"`
+	Info            string           `json:"info"`
+	TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
 }
 
 func (this *UpdateAlertStateCommand) IsValidState() bool {
@@ -27,9 +29,11 @@ func (this *UpdateAlertStateCommand) IsValidState() bool {
 // Commands
 
 type UpdateAlertStateCommand struct {
-	AlertId  int64  `json:"alertId" binding:"Required"`
-	NewState string `json:"newState" binding:"Required"`
-	Info     string `json:"info"`
+	AlertId         int64            `json:"alertId" binding:"Required"`
+	OrgId           int64            `json:"orgId" binding:"Required"`
+	NewState        string           `json:"newState" binding:"Required"`
+	Info            string           `json:"info"`
+	TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
 
 	Result *Alert
 }
@@ -42,3 +46,10 @@ type GetAlertsStateQuery struct {
 
 	Result *[]AlertState
 }
+
+type GetLastAlertStateQuery struct {
+	AlertId int64
+	OrgId   int64
+
+	Result *AlertState
+}

pkg/models/dashboards.go

@@ -151,7 +151,7 @@ type GetDashboardTagsQuery struct {
 
 type GetDashboardsQuery struct {
 	DashboardIds []int64
-	Result       *[]Dashboard
+	Result       []*Dashboard
 }
 
 type GetDashboardSlugByIdQuery struct {

pkg/models/notifications.go

@@ -12,6 +12,13 @@ type SendEmailCommand struct {
 	Info     string
 }
 
+type SendWebhook struct {
+	Url      string
+	User     string
+	Password string
+	Body     string
+}
+
 type SendResetPasswordEmailCommand struct {
 	User *User
 }

pkg/services/alerting/alert_rule.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/services/alerting/transformers"
@@ -26,6 +27,8 @@ type AlertRule struct {
 	Transform       string
 	TransformParams simplejson.Json
 	Transformer     transformers.Transformer
+
+	NotificationGroups []int64
 }
 
 var (
@@ -61,7 +64,18 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
 	model.State = ruleDef.State
 	model.Frequency = ruleDef.Frequency
 
-	critical := ruleDef.Settings.Get("critical")
+	ngs := ruleDef.Settings.Get("notificationGroups").MustString()
+	var ids []int64
+	for _, v := range strings.Split(ngs, ",") {
+		id, err := strconv.Atoi(v)
+		if err == nil {
+			ids = append(ids, int64(id))
+		}
+	}
+
+	model.NotificationGroups = ids
+
+	critical := ruleDef.Settings.Get("crit")
 	model.Critical = Level{
 		Operator: critical.Get("op").MustString(),
 		Value:    critical.Get("value").MustFloat64(),
@@ -74,6 +88,10 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
 	}
 
 	model.Transform = ruleDef.Settings.Get("transform").Get("type").MustString()
+	if model.Transform == "" {
+		return nil, fmt.Errorf("missing transform")
+	}
+
 	model.TransformParams = *ruleDef.Settings.Get("transform")
 
 	if model.Transform == "aggregation" {
@@ -87,7 +105,6 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
 		DatasourceId: query.Get("datasourceId").MustInt64(),
 		From:         query.Get("from").MustString(),
 		To:           query.Get("to").MustString(),
-		Aggregator:   query.Get("agg").MustString(),
 	}
 
 	if model.Query.Query == "" {

pkg/services/alerting/alert_rule_test.go

@@ -38,7 +38,7 @@ func TestAlertRuleModel(t *testing.T) {
 				"description": "desc2",
 				"handler": 0,
 				"enabled": true,
-				"critical": {
+				"crit": {
 					"value": 20,
 					"op": ">"
 				},
@@ -55,7 +55,7 @@ func TestAlertRuleModel(t *testing.T) {
 					"datasourceId": 1
 				},
 				"transform": {
-					"method": "avg",
+					"type": "avg",
 					"name": "aggregation"
 				}
 			}
@@ -75,11 +75,12 @@ func TestAlertRuleModel(t *testing.T) {
 			alertRule, err := NewAlertRuleFromDBModel(alert)
 
 			So(err, ShouldBeNil)
-			So(alertRule.Critical.Operator, ShouldEqual, ">")
-			So(alertRule.Critical.Value, ShouldEqual, 20)
 
 			So(alertRule.Warning.Operator, ShouldEqual, ">")
 			So(alertRule.Warning.Value, ShouldEqual, 10)
+
+			So(alertRule.Critical.Operator, ShouldEqual, ">")
+			So(alertRule.Critical.Value, ShouldEqual, 20)
 		})
 	})
 }

pkg/services/alerting/datasources/backends.go

@@ -1,3 +0,0 @@
-package datasources
-
-// GetSeries returns timeseries data from the datasource

pkg/services/alerting/datasources/graphite.go

@@ -1,80 +0,0 @@
-package datasources
-
-// import (
-// 	"bytes"
-// 	"encoding/json"
-// 	"fmt"
-// 	"io/ioutil"
-// 	"net/http"
-// 	"net/url"
-// 	"strconv"
-// 	"time"
-//
-// 	"github.com/grafana/grafana/pkg/components/simplejson"
-// 	"github.com/grafana/grafana/pkg/log"
-// 	m "github.com/grafana/grafana/pkg/models"
-// 	"github.com/grafana/grafana/pkg/util"
-// )
-//
-// type GraphiteClient struct{}
-//
-// type GraphiteSerie struct {
-// 	Datapoints [][2]float64
-// 	Target     string
-// }
-//
-// var DefaultClient = &http.Client{
-// 	Timeout: time.Minute,
-// }
-//
-// type GraphiteResponse []GraphiteSerie
-//
-// func (client GraphiteClient) GetSeries(rule m.AlertJob, datasource m.DataSource) (m.TimeSeriesSlice, error) {
-// 	v := url.Values{
-// 		"format": []string{"json"},
-// 		"target": []string{getTargetFromRule(rule.Rule)},
-// 		"until":  []string{"now"},
-// 		"from":   []string{"-" + strconv.Itoa(rule.Rule.QueryRange) + "s"},
-// 	}
-//
-// 	log.Trace("Graphite: sending request with querystring: ", v.Encode())
-//
-// 	req, err := http.NewRequest("POST", datasource.Url+"/render", nil)
-//
-// 	if err != nil {
-// 		return nil, fmt.Errorf("Could not create request")
-// 	}
-//
-// 	req.Body = ioutil.NopCloser(bytes.NewReader([]byte(v.Encode())))
-//
-// 	if datasource.BasicAuth {
-// 		req.Header.Add("Authorization", util.GetBasicAuthHeader(datasource.User, datasource.Password))
-// 	}
-//
-// 	res, err := DefaultClient.Do(req)
-//
-// 	if err != nil {
-// 		return nil, err
-// 	}
-//
-// 	if res.StatusCode != http.StatusOK {
-// 		return nil, fmt.Errorf("expected httpstatus 200, found %d", res.StatusCode)
-// 	}
-//
-// 	response := GraphiteResponse{}
-//
-// 	json.NewDecoder(res.Body).Decode(&response)
-//
-// 	var timeSeries []*m.TimeSeries
-// 	for _, v := range response {
-// 		timeSeries = append(timeSeries, m.NewTimeSeries(v.Target, v.Datapoints))
-// 	}
-//
-// 	return timeSeries, nil
-// }
-//
-// func getTargetFromRule(rule m.AlertRule) string {
-// 	json, _ := simplejson.NewJson([]byte(rule.Query))
-//
-// 	return json.Get("target").MustString()
-// }

pkg/services/alerting/engine.go

@@ -5,32 +5,32 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
-	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
-	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
 )
 
 type Engine struct {
-	execQueue   chan *AlertJob
-	resultQueue chan *AlertResult
-	clock       clock.Clock
-	ticker      *Ticker
-	scheduler   Scheduler
-	handler     AlertingHandler
-	ruleReader  RuleReader
-	log         log.Logger
+	execQueue       chan *AlertJob
+	resultQueue     chan *AlertResult
+	clock           clock.Clock
+	ticker          *Ticker
+	scheduler       Scheduler
+	handler         AlertingHandler
+	ruleReader      RuleReader
+	log             log.Logger
+	responseHandler ResultHandler
 }
 
 func NewEngine() *Engine {
 	e := &Engine{
-		ticker:      NewTicker(time.Now(), time.Second*0, clock.New()),
-		execQueue:   make(chan *AlertJob, 1000),
-		resultQueue: make(chan *AlertResult, 1000),
-		scheduler:   NewScheduler(),
-		handler:     NewHandler(),
-		ruleReader:  NewRuleReader(),
-		log:         log.New("alerting.engine"),
+		ticker:          NewTicker(time.Now(), time.Second*0, clock.New()),
+		execQueue:       make(chan *AlertJob, 1000),
+		resultQueue:     make(chan *AlertResult, 1000),
+		scheduler:       NewScheduler(),
+		handler:         NewHandler(),
+		ruleReader:      NewRuleReader(),
+		log:             log.New("alerting.engine"),
+		responseHandler: NewResultHandler(),
 	}
 
 	return e
@@ -52,7 +52,7 @@ func (e *Engine) Stop() {
 func (e *Engine) alertingTicker() {
 	defer func() {
 		if err := recover(); err != nil {
-			e.log.Error("Scheduler Panic, stopping...", "error", err, "stack", log.Stack(1))
+			e.log.Error("Scheduler Panic: stopping alertingTicker", "error", err, "stack", log.Stack(1))
 		}
 	}()
 
@@ -73,6 +73,12 @@ func (e *Engine) alertingTicker() {
 }
 
 func (e *Engine) execDispatch() {
+	defer func() {
+		if err := recover(); err != nil {
+			e.log.Error("Scheduler Panic: stopping executor", "error", err, "stack", log.Stack(1))
+		}
+	}()
+
 	for job := range e.execQueue {
 		log.Trace("Alerting: engine:execDispatch() starting job %s", job.Rule.Name)
 		job.Running = true
@@ -89,10 +95,11 @@ func (e *Engine) executeJob(job *AlertJob) {
 	select {
 	case <-time.After(time.Second * 5):
 		e.resultQueue <- &AlertResult{
-			State:    alertstates.Pending,
-			Duration: float64(time.Since(now).Nanoseconds()) / float64(1000000),
-			Error:    fmt.Errorf("Timeout"),
-			AlertJob: job,
+			State:         alertstates.Pending,
+			Duration:      float64(time.Since(now).Nanoseconds()) / float64(1000000),
+			Error:         fmt.Errorf("Timeout"),
+			AlertJob:      job,
+			ExeuctionTime: time.Now(),
 		}
 		e.log.Debug("Job Execution timeout", "alertRuleId", job.Rule.Id)
 	case result := <-resultChan:
@@ -103,6 +110,12 @@ func (e *Engine) executeJob(job *AlertJob) {
 }
 
 func (e *Engine) resultHandler() {
+	defer func() {
+		if err := recover(); err != nil {
+			e.log.Error("Engine Panic, stopping resultHandler", "error", err, "stack", log.Stack(1))
+		}
+	}()
+
 	for result := range e.resultQueue {
 		e.log.Debug("Alert Rule Result", "ruleId", result.AlertJob.Rule.Id, "state", result.State, "value", result.ActualValue, "retry", result.AlertJob.RetryCount)
 
@@ -119,23 +132,11 @@ func (e *Engine) resultHandler() {
 
 				result.State = alertstates.Critical
 				result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error)
-				e.saveState(result)
+				e.responseHandler.Handle(result)
 			}
 		} else {
 			result.AlertJob.ResetRetry()
-			e.saveState(result)
+			e.responseHandler.Handle(result)
 		}
 	}
 }
-
-func (e *Engine) saveState(result *AlertResult) {
-	cmd := &m.UpdateAlertStateCommand{
-		AlertId:  result.AlertJob.Rule.Id,
-		NewState: result.State,
-		Info:     result.Description,
-	}
-
-	if err := bus.Dispatch(cmd); err != nil {
-		e.log.Error("Failed to save state", "error", err)
-	}
-}

pkg/services/alerting/extractor.go

@@ -23,28 +23,28 @@ func NewDashAlertExtractor(dash *m.Dashboard, orgId int64) *DashAlertExtractor {
 	}
 }
 
-func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (int64, error) {
+func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (*m.DataSource, error) {
 	if dsName == "" {
 		query := &m.GetDataSourcesQuery{OrgId: e.OrgId}
 		if err := bus.Dispatch(query); err != nil {
-			return 0, err
+			return nil, err
 		} else {
 			for _, ds := range query.Result {
 				if ds.IsDefault {
-					return ds.Id, nil
+					return ds, nil
 				}
 			}
 		}
 	} else {
 		query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgId}
 		if err := bus.Dispatch(query); err != nil {
-			return 0, err
+			return nil, err
 		} else {
-			return query.Result.Id, nil
+			return query.Result, nil
 		}
 	}
 
-	return 0, errors.New("Could not find datasource id for " + dsName)
+	return nil, errors.New("Could not find datasource id for " + dsName)
 }
 
 func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
@@ -94,10 +94,11 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
 						dsName = panel.Get("datasource").MustString()
 					}
 
-					if datasourceId, err := e.lookupDatasourceId(dsName); err != nil {
+					if datasource, err := e.lookupDatasourceId(dsName); err != nil {
 						return nil, err
 					} else {
-						valueQuery.SetPath([]string{"datasourceId"}, datasourceId)
+						valueQuery.SetPath([]string{"datasourceId"}, datasource.Id)
+						valueQuery.SetPath([]string{"datasourceType"}, datasource.Type)
 					}
 
 					targetQuery := target.Get("target").MustString()

pkg/services/alerting/extractor_test.go

@@ -52,8 +52,8 @@ func TestAlertRuleExtraction(t *testing.T) {
               "to": "now"
             },
             "transform": {
-              "method": "avg",
-              "type": "aggregation"
+              "type": "avg",
+              "name": "aggregation"
             },
             "warn": {
               "value": 10,
@@ -87,7 +87,7 @@ func TestAlertRuleExtraction(t *testing.T) {
               "to": "now"
             },
             "transform": {
-              "method": "avg",
+              "type": "avg",
               "name": "aggregation"
             },
             "warn": {

pkg/services/alerting/handler.go

@@ -2,6 +2,7 @@ package alerting
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
@@ -28,9 +29,10 @@ func (e *HandlerImpl) Execute(job *AlertJob, resultQueue chan *AlertResult) {
 	timeSeries, err := e.executeQuery(job)
 	if err != nil {
 		resultQueue <- &AlertResult{
-			Error:    err,
-			State:    alertstates.Pending,
-			AlertJob: job,
+			Error:         err,
+			State:         alertstates.Pending,
+			AlertJob:      job,
+			ExeuctionTime: time.Now(),
 		}
 	}
 
@@ -102,17 +104,20 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice)
 		transformedValue, _ := rule.Transformer.Transform(serie)
 
 		critResult := evalCondition(rule.Critical, transformedValue)
-		e.log.Debug("Alert execution Crit", "name", serie.Name, "transformedValue", transformedValue, "operator", rule.Critical.Operator, "level", rule.Critical.Value, "result", critResult)
+		condition2 := fmt.Sprintf("%v %s %v ", transformedValue, rule.Critical.Operator, rule.Critical.Value)
+		e.log.Debug("Alert execution Crit", "name", serie.Name, "condition", condition2, "result", critResult)
 		if critResult {
 			triggeredAlert = append(triggeredAlert, &TriggeredAlert{
 				State:       alertstates.Critical,
 				ActualValue: transformedValue,
 				Name:        serie.Name,
 			})
+			continue
 		}
 
 		warnResult := evalCondition(rule.Warning, transformedValue)
-		e.log.Debug("Alert execution Warn", "name", serie.Name, "transformedValue", transformedValue, "operator", rule.Warning.Operator, "level", rule.Warning.Value, "result", warnResult)
+		condition := fmt.Sprintf("%v %s %v ", transformedValue, rule.Warning.Operator, rule.Warning.Value)
+		e.log.Debug("Alert execution Warn", "name", serie.Name, "condition", condition, "result", warnResult)
 		if warnResult {
 			triggeredAlert = append(triggeredAlert, &TriggeredAlert{
 				State:       alertstates.Warn,
@@ -123,7 +128,6 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice)
 	}
 
 	executionState := alertstates.Ok
-	description := ""
 	for _, raised := range triggeredAlert {
 		if raised.State == alertstates.Critical {
 			executionState = alertstates.Critical
@@ -132,9 +136,7 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice)
 		if executionState != alertstates.Critical && raised.State == alertstates.Warn {
 			executionState = alertstates.Warn
 		}
-
-		description += fmt.Sprintf(descriptionFmt, raised.ActualValue, raised.Name)
 	}
 
-	return &AlertResult{State: executionState, Description: description, TriggeredAlerts: triggeredAlert}
+	return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert, ExeuctionTime: time.Now()}
 }

pkg/services/alerting/interfaces.go

@@ -10,3 +10,7 @@ type Scheduler interface {
 	Tick(time time.Time, execQueue chan *AlertJob)
 	Update(rules []*AlertRule)
 }
+
+type Notifier interface {
+	Notify(alertResult *AlertResult)
+}

pkg/services/alerting/models.go

@@ -1,5 +1,7 @@
 package alerting
 
+import "time"
+
 type AlertJob struct {
 	Offset     int64
 	Delay      bool
@@ -28,6 +30,7 @@ type AlertResult struct {
 	Description     string
 	Error           error
 	AlertJob        *AlertJob
+	ExeuctionTime   time.Time
 }
 
 type TriggeredAlert struct {
@@ -44,7 +47,6 @@ type Level struct {
 type AlertQuery struct {
 	Query        string
 	DatasourceId int64
-	Aggregator   string
 	From         string
 	To           string
 }

pkg/services/alerting/notifier.go

@@ -1 +1,208 @@
 package alerting
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type NotifierImpl struct {
+	log              log.Logger
+	getNotifications func(orgId int64, notificationGroups []int64) []*Notification
+}
+
+func NewNotifier() *NotifierImpl {
+	log := log.New("alerting.notifier")
+	return &NotifierImpl{
+		log:              log,
+		getNotifications: buildGetNotifiers(log),
+	}
+}
+
+func (n NotifierImpl) ShouldDispath(alertResult *AlertResult, notifier *Notification) bool {
+	warn := alertResult.State == alertstates.Warn && notifier.SendWarning
+	crit := alertResult.State == alertstates.Critical && notifier.SendCritical
+	return (warn || crit) || alertResult.State == alertstates.Ok
+}
+
+func (n *NotifierImpl) Notify(alertResult *AlertResult) {
+	notifiers := n.getNotifications(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups)
+
+	for _, notifier := range notifiers {
+		if n.ShouldDispath(alertResult, notifier) {
+			n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type)
+			go notifier.Notifierr.Dispatch(alertResult)
+		}
+	}
+}
+
+type Notification struct {
+	Name         string
+	Type         string
+	SendWarning  bool
+	SendCritical bool
+
+	Notifierr NotificationDispatcher
+}
+
+type EmailNotifier struct {
+	To  string
+	log log.Logger
+}
+
+func (this *EmailNotifier) Dispatch(alertResult *AlertResult) {
+	this.log.Info("Sending email")
+	grafanaUrl := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
+	if setting.AppSubUrl != "" {
+		grafanaUrl += "/" + setting.AppSubUrl
+	}
+
+	query := &m.GetDashboardsQuery{
+		DashboardIds: []int64{alertResult.AlertJob.Rule.DashboardId},
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		this.log.Error("Failed to load dashboard", "error", err)
+		return
+	}
+
+	if len(query.Result) != 1 {
+		this.log.Error("Can only support one dashboard", "result", len(query.Result))
+		return
+	}
+
+	dashboard := query.Result[0]
+
+	panelId := strconv.Itoa(int(alertResult.AlertJob.Rule.PanelId))
+
+	//TODO: get from alertrule and transforms to seconds
+	from := "1466169458375"
+	to := "1466171258375"
+
+	renderUrl := fmt.Sprintf("%s/render/dashboard-solo/db/%s?from=%s&to=%s&panelId=%s&width=1000&height=500", grafanaUrl, dashboard.Slug, from, to, panelId)
+	cmd := &m.SendEmailCommand{
+		Data: map[string]interface{}{
+			"Name":            "Name",
+			"State":           alertResult.State,
+			"Description":     alertResult.Description,
+			"TriggeredAlerts": alertResult.TriggeredAlerts,
+			"DashboardLink":   grafanaUrl + "/dashboard/db/" + dashboard.Slug,
+			"AlertPageUrl":    grafanaUrl + "/alerting",
+			"DashboardImage":  renderUrl,
+		},
+		To:       []string{this.To},
+		Template: "alert_notification.html",
+	}
+
+	err := bus.Dispatch(cmd)
+	if err != nil {
+		this.log.Error("Could not send alert notification as email", "error", err)
+	}
+}
+
+type WebhookNotifier struct {
+	Url      string
+	User     string
+	Password string
+	log      log.Logger
+}
+
+func (this *WebhookNotifier) Dispatch(alertResult *AlertResult) {
+	this.log.Info("Sending webhook")
+
+	bodyJSON := simplejson.New()
+	bodyJSON.Set("name", alertResult.AlertJob.Rule.Name)
+	bodyJSON.Set("state", alertResult.State)
+	bodyJSON.Set("trigged", alertResult.TriggeredAlerts)
+
+	body, _ := bodyJSON.MarshalJSON()
+
+	cmd := &m.SendWebhook{
+		Url:      this.Url,
+		User:     this.User,
+		Password: this.Password,
+		Body:     string(body),
+	}
+
+	bus.Dispatch(cmd)
+}
+
+type NotificationDispatcher interface {
+	Dispatch(alertResult *AlertResult)
+}
+
+func buildGetNotifiers(log log.Logger) func(orgId int64, notificationGroups []int64) []*Notification {
+	return func(orgId int64, notificationGroups []int64) []*Notification {
+		query := &m.GetAlertNotificationQuery{
+			OrgID:                orgId,
+			Ids:                  notificationGroups,
+			IncludeAlwaysExecute: true,
+		}
+		err := bus.Dispatch(query)
+		if err != nil {
+			log.Error("Failed to read notifications", "error", err)
+		}
+
+		var result []*Notification
+		log.Info("notifiriring", "count", len(query.Result), "groups", notificationGroups)
+		for _, notification := range query.Result {
+			not, err := NewNotificationFromDBModel(notification)
+			if err == nil {
+				result = append(result, not)
+			} else {
+				log.Error("Failed to read notification model", "error", err)
+			}
+		}
+
+		return result
+	}
+}
+
+func NewNotificationFromDBModel(model *m.AlertNotification) (*Notification, error) {
+	notifier, err := createNotifier(model.Type, model.Settings)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notification{
+		Name:         model.Name,
+		Type:         model.Type,
+		Notifierr:    notifier,
+		SendCritical: model.Settings.Get("sendCrit").MustBool(),
+		SendWarning:  model.Settings.Get("sendWarn").MustBool(),
+	}, nil
+}
+
+var createNotifier = func(notificationType string, settings *simplejson.Json) (NotificationDispatcher, error) {
+	if notificationType == "email" {
+		to := settings.Get("to").MustString()
+
+		if to == "" {
+			return nil, fmt.Errorf("Could not find to propertie in settings")
+		}
+
+		return &EmailNotifier{
+			To:  to,
+			log: log.New("alerting.notification.email"),
+		}, nil
+	}
+
+	url := settings.Get("url").MustString()
+	if url == "" {
+		return nil, fmt.Errorf("Could not find url propertie in settings")
+	}
+
+	return &WebhookNotifier{
+		Url:      url,
+		User:     settings.Get("user").MustString(),
+		Password: settings.Get("password").MustString(),
+		log:      log.New("alerting.notification.webhook"),
+	}, nil
+}

pkg/services/alerting/notifier_test.go

@@ -0,0 +1,125 @@
+package alerting
+
+import (
+	"testing"
+
+	"reflect"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAlertNotificationExtraction(t *testing.T) {
+	Convey("Notifier tests", t, func() {
+		Convey("rules for sending notifications", func() {
+			dummieNotifier := NotifierImpl{}
+
+			result := &AlertResult{
+				State: alertstates.Critical,
+			}
+
+			notifier := &Notification{
+				Name:         "Test Notifier",
+				Type:         "TestType",
+				SendCritical: true,
+				SendWarning:  true,
+			}
+
+			Convey("Should send notification", func() {
+				So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeTrue)
+			})
+
+			Convey("warn:false and state:warn should not send", func() {
+				result.State = alertstates.Warn
+				notifier.SendWarning = false
+				So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeFalse)
+			})
+		})
+
+		Convey("Parsing alert notification from settings", func() {
+			Convey("Parsing email", func() {
+				Convey("empty settings should return error", func() {
+					json := `{ }`
+
+					settingsJSON, _ := simplejson.NewJson([]byte(json))
+					model := &m.AlertNotification{
+						Name:     "ops",
+						Type:     "email",
+						Settings: settingsJSON,
+					}
+
+					_, err := NewNotificationFromDBModel(model)
+					So(err, ShouldNotBeNil)
+				})
+
+				Convey("from settings", func() {
+					json := `
+				{
+					"to": "ops@grafana.org"
+				}`
+
+					settingsJSON, _ := simplejson.NewJson([]byte(json))
+					model := &m.AlertNotification{
+						Name:     "ops",
+						Type:     "email",
+						Settings: settingsJSON,
+					}
+
+					not, err := NewNotificationFromDBModel(model)
+
+					So(err, ShouldBeNil)
+					So(not.Name, ShouldEqual, "ops")
+					So(not.Type, ShouldEqual, "email")
+					So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier")
+
+					email := not.Notifierr.(*EmailNotifier)
+					So(email.To, ShouldEqual, "ops@grafana.org")
+				})
+			})
+
+			Convey("Parsing webhook", func() {
+				Convey("empty settings should return error", func() {
+					json := `{ }`
+
+					settingsJSON, _ := simplejson.NewJson([]byte(json))
+					model := &m.AlertNotification{
+						Name:     "ops",
+						Type:     "webhook",
+						Settings: settingsJSON,
+					}
+
+					_, err := NewNotificationFromDBModel(model)
+					So(err, ShouldNotBeNil)
+				})
+
+				Convey("from settings", func() {
+					json := `
+				{
+					"url": "http://localhost:3000",
+					"username": "username",
+					"password": "password"
+				}`
+
+					settingsJSON, _ := simplejson.NewJson([]byte(json))
+					model := &m.AlertNotification{
+						Name:     "slack",
+						Type:     "webhook",
+						Settings: settingsJSON,
+					}
+
+					not, err := NewNotificationFromDBModel(model)
+
+					So(err, ShouldBeNil)
+					So(not.Name, ShouldEqual, "slack")
+					So(not.Type, ShouldEqual, "webhook")
+					So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier")
+
+					webhook := not.Notifierr.(*WebhookNotifier)
+					So(webhook.Url, ShouldEqual, "http://localhost:3000")
+				})
+			})
+		})
+	})
+}

pkg/services/alerting/result_handler.go

@@ -0,0 +1,70 @@
+package alerting
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+)
+
+type ResultHandler interface {
+	Handle(result *AlertResult)
+}
+
+type ResultHandlerImpl struct {
+	notifier Notifier
+	log      log.Logger
+}
+
+func NewResultHandler() *ResultHandlerImpl {
+	return &ResultHandlerImpl{
+		log:      log.New("alerting.responseHandler"),
+		notifier: NewNotifier(),
+	}
+}
+
+func (handler *ResultHandlerImpl) Handle(result *AlertResult) {
+	if handler.shouldUpdateState(result) {
+		cmd := &m.UpdateAlertStateCommand{
+			AlertId:         result.AlertJob.Rule.Id,
+			NewState:        result.State,
+			Info:            result.Description,
+			OrgId:           result.AlertJob.Rule.OrgId,
+			TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts),
+		}
+
+		if err := bus.Dispatch(cmd); err != nil {
+			handler.log.Error("Failed to save state", "error", err)
+		}
+
+		handler.log.Debug("will notify about new state", "new state", result.State)
+		handler.notifier.Notify(result)
+	}
+}
+
+func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool {
+	query := &m.GetLastAlertStateQuery{
+		AlertId: result.AlertJob.Rule.Id,
+		OrgId:   result.AlertJob.Rule.OrgId,
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		log.Error2("Failed to read last alert state", "error", err)
+		return false
+	}
+
+	if query.Result == nil {
+		return true
+	}
+
+	//now := time.Now()
+	//olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15))
+	lastExecution := query.Result.Created
+	asdf := result.ExeuctionTime.Add(time.Minute * -15)
+	olderThen15Min := lastExecution.Before(asdf)
+	changedState := query.Result.NewState != result.State
+
+	return changedState || olderThen15Min
+}

pkg/services/alerting/result_handler_test.go

@@ -0,0 +1,59 @@
+package alerting
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAlertResultHandler(t *testing.T) {
+	Convey("Test result Handler", t, func() {
+		resultHandler := ResultHandlerImpl{}
+		mockResult := &AlertResult{
+			State: alertstates.Ok,
+			AlertJob: &AlertJob{
+				Rule: &AlertRule{
+					Id:    1,
+					OrgId: 1,
+				},
+			},
+		}
+		mockAlertState := &m.AlertState{}
+		bus.ClearBusHandlers()
+		bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error {
+			query.Result = mockAlertState
+			return nil
+		})
+
+		Convey("Should update", func() {
+
+			Convey("when no earlier alert state", func() {
+				mockAlertState = nil
+				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
+			})
+
+			Convey("alert state have changed", func() {
+				mockAlertState = &m.AlertState{
+					NewState: alertstates.Critical,
+				}
+				mockResult.State = alertstates.Ok
+				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
+			})
+
+			Convey("last alert state was 15min ago", func() {
+				now := time.Now()
+				mockAlertState = &m.AlertState{
+					NewState: alertstates.Critical,
+					Created:  now.Add(time.Minute * -30),
+				}
+				mockResult.State = alertstates.Critical
+				mockResult.ExeuctionTime = time.Now()
+				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
+			})
+		})
+	})
+}

pkg/services/notifications/notifications.go

@@ -23,11 +23,14 @@ var tmplWelcomeOnSignUp = "welcome_on_signup.html"
 
 func Init() error {
 	initMailQueue()
+	initWebhookQueue()
 
 	bus.AddHandler("email", sendResetPasswordEmail)
 	bus.AddHandler("email", validateResetPasswordCode)
 	bus.AddHandler("email", sendEmailCommandHandler)
 
+	bus.AddHandler("webhook", sendWebhook)
+
 	bus.AddEventListener(signUpStartedHandler)
 	bus.AddEventListener(signUpCompletedHandler)
 
@@ -53,6 +56,17 @@ func Init() error {
 	return nil
 }
 
+func sendWebhook(cmd *m.SendWebhook) error {
+	addToWebhookQueue(&Webhook{
+		Url:      cmd.Url,
+		User:     cmd.User,
+		Password: cmd.Password,
+		Body:     cmd.Body,
+	})
+
+	return nil
+}
+
 func subjectTemplateFunc(obj map[string]interface{}, value string) string {
 	obj["value"] = value
 	return ""

pkg/services/notifications/notifications_test.go

@@ -9,6 +9,12 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+type testTriggeredAlert struct {
+	ActualValue float64
+	Name        string
+	State       string
+}
+
 func TestNotifications(t *testing.T) {
 
 	Convey("Given the notifications service", t, func() {
@@ -34,6 +40,84 @@ func TestNotifications(t *testing.T) {
 			So(sentMsg.Subject, ShouldEqual, "Reset your Grafana password - asd@asd.com")
 			So(sentMsg.Body, ShouldNotContainSubstring, "Subject")
 		})
-	})
 
+		Convey("Alert notifications", func() {
+			Convey("When sending reset email password", func() {
+				cmd := &m.SendEmailCommand{
+					Data: map[string]interface{}{
+						"Name":           "Name",
+						"State":          "Critical",
+						"Description":    "Description",
+						"DashboardLink":  "http://localhost:3000/dashboard/db/alerting",
+						"AlertPageUrl":   "http://localhost:3000/alerting",
+						"DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500",
+						"TriggeredAlerts": []testTriggeredAlert{
+							{Name: "desktop", State: "Critical", ActualValue: 13},
+							{Name: "mobile", State: "Warn", ActualValue: 5},
+						},
+					},
+					To:       []string{"asd@asd.com "},
+					Template: "alert_notification.html",
+				}
+
+				err := sendEmailCommandHandler(cmd)
+				So(err, ShouldBeNil)
+
+				So(sentMsg.Body, ShouldContainSubstring, "Alertstate: Critical")
+				So(sentMsg.Body, ShouldContainSubstring, "http://localhost:3000/dashboard/db/alerting")
+				So(sentMsg.Body, ShouldContainSubstring, "Critical")
+				So(sentMsg.Body, ShouldContainSubstring, "Warn")
+				So(sentMsg.Body, ShouldContainSubstring, "mobile")
+				So(sentMsg.Body, ShouldContainSubstring, "desktop")
+				So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Critical ] ")
+			})
+
+			Convey("given critical", func() {
+				cmd := &m.SendEmailCommand{
+					Data: map[string]interface{}{
+						"Name":           "Name",
+						"State":          "Warn",
+						"Description":    "Description",
+						"DashboardLink":  "http://localhost:3000/dashboard/db/alerting",
+						"DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=1&width=1000&height=500",
+						"AlertPageUrl":   "http://localhost:3000/alerting",
+						"TriggeredAlerts": []testTriggeredAlert{
+							{Name: "desktop", State: "Critical", ActualValue: 13},
+							{Name: "mobile", State: "Warn", ActualValue: 5},
+						},
+					},
+					To:       []string{"asd@asd.com "},
+					Template: "alert_notification.html",
+				}
+
+				err := sendEmailCommandHandler(cmd)
+				So(err, ShouldBeNil)
+				So(sentMsg.Body, ShouldContainSubstring, "Alertstate: Warn")
+				So(sentMsg.Body, ShouldContainSubstring, "http://localhost:3000/dashboard/db/alerting")
+				So(sentMsg.Body, ShouldContainSubstring, "Critical")
+				So(sentMsg.Body, ShouldContainSubstring, "Warn")
+				So(sentMsg.Body, ShouldContainSubstring, "mobile")
+				So(sentMsg.Body, ShouldContainSubstring, "desktop")
+				So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Warn ]")
+			})
+
+			Convey("given ok", func() {
+				cmd := &m.SendEmailCommand{
+					Data: map[string]interface{}{
+						"Name":          "Name",
+						"State":         "Ok",
+						"Description":   "Description",
+						"DashboardLink": "http://localhost:3000/dashboard/db/alerting",
+						"AlertPageUrl":  "http://localhost:3000/alerting",
+					},
+					To:       []string{"asd@asd.com "},
+					Template: "alert_notification.html",
+				}
+
+				err := sendEmailCommandHandler(cmd)
+				So(err, ShouldBeNil)
+				So(sentMsg.Subject, ShouldContainSubstring, "Grafana Alert: [ Ok ]")
+			})
+		})
+	})
 }

pkg/services/notifications/send_email_integration_test.go

@@ -0,0 +1,53 @@
+package notifications
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/bus"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestEmailIntegrationTest(t *testing.T) {
+	SkipConvey("Given the notifications service", t, func() {
+		bus.ClearBusHandlers()
+
+		setting.StaticRootPath = "../../../public/"
+		setting.Smtp.Enabled = true
+		setting.Smtp.TemplatesPattern = "emails/*.html"
+		setting.Smtp.FromAddress = "from@address.com"
+
+		err := Init()
+		So(err, ShouldBeNil)
+
+		addToMailQueue = func(msg *Message) {
+			ioutil.WriteFile("../../../tmp/test_email.html", []byte(msg.Body), 0777)
+		}
+
+		Convey("When sending reset email password", func() {
+			cmd := &m.SendEmailCommand{
+
+				Data: map[string]interface{}{
+					"Name":           "Name",
+					"State":          "Critical",
+					"Description":    "Description",
+					"DashboardLink":  "http://localhost:3000/dashboard/db/alerting",
+					"AlertPageUrl":   "http://localhost:3000/alerting",
+					"DashboardImage": "http://localhost:3000/render/dashboard-solo/db/alerting?from=1466169458375&to=1466171258375&panelId=3&width=1000&height=500",
+
+					"TriggeredAlerts": []testTriggeredAlert{
+						{Name: "desktop", State: "Critical", ActualValue: 13},
+						{Name: "mobile", State: "Warn", ActualValue: 5},
+					},
+				},
+				To:       []string{"asd@asd.com "},
+				Template: "alert_notification.html",
+			}
+
+			err := sendEmailCommandHandler(cmd)
+			So(err, ShouldBeNil)
+		})
+	})
+}

pkg/services/notifications/webhook.go

@@ -0,0 +1,66 @@
+package notifications
+
+import (
+	"bytes"
+	"net/http"
+	"time"
+
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+type Webhook struct {
+	Url      string
+	User     string
+	Password string
+	Body     string
+}
+
+var webhookQueue chan *Webhook
+var webhookLog log.Logger
+
+func initWebhookQueue() {
+	webhookLog = log.New("notifications.webhook")
+	webhookQueue = make(chan *Webhook, 10)
+	go processWebhookQueue()
+}
+
+func processWebhookQueue() {
+	for {
+		select {
+		case webhook := <-webhookQueue:
+			err := sendWebRequest(webhook)
+
+			if err != nil {
+				webhookLog.Error("Failed to send webrequest ", "error", err)
+			}
+		}
+	}
+}
+
+func sendWebRequest(webhook *Webhook) error {
+	client := http.Client{
+		Timeout: time.Duration(3 * time.Second),
+	}
+
+	request, err := http.NewRequest("POST", webhook.Url, bytes.NewReader([]byte(webhook.Body)))
+	if webhook.User != "" && webhook.Password != "" {
+		request.Header.Add("Authorization", util.GetBasicAuthHeader(webhook.User, webhook.Password))
+	}
+
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return nil
+}
+
+var addToWebhookQueue = func(msg *Webhook) {
+	webhookQueue <- msg
+}

pkg/services/sqlstore/alert_notification.go

@@ -0,0 +1,185 @@
+package sqlstore
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/go-xorm/xorm"
+	"github.com/grafana/grafana/pkg/bus"
+	m "github.com/grafana/grafana/pkg/models"
+)
+
+func init() {
+	bus.AddHandler("sql", AlertNotificationQuery)
+	bus.AddHandler("sql", CreateAlertNotificationCommand)
+	bus.AddHandler("sql", UpdateAlertNotification)
+	bus.AddHandler("sql", DeleteAlertNotification)
+}
+
+func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
+	return inTransaction(func(sess *xorm.Session) error {
+		sql := "DELETE FROM alert_notification WHERE alert_notification.org_id = ? AND alert_notification.id = ?"
+		_, err := sess.Exec(sql, cmd.OrgId, cmd.Id)
+
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AlertNotificationQuery(query *m.GetAlertNotificationQuery) error {
+	return getAlertNotifications(query, x.NewSession())
+}
+
+func getAlertNotifications(query *m.GetAlertNotificationQuery, sess *xorm.Session) error {
+	var sql bytes.Buffer
+	params := make([]interface{}, 0)
+
+	sql.WriteString(`SELECT
+	   					  alert_notification.id,
+	   					  alert_notification.org_id,
+	   					  alert_notification.name,
+	                      alert_notification.type,
+	   					  alert_notification.created,
+	                      alert_notification.updated,
+	                      alert_notification.settings,
+						  alert_notification.always_execute
+	   					  FROM alert_notification
+	   					  `)
+
+	sql.WriteString(` WHERE alert_notification.org_id = ?`)
+	params = append(params, query.OrgID)
+
+	if query.Name != "" {
+		sql.WriteString(` AND alert_notification.name = ?`)
+		params = append(params, query.Name)
+	}
+
+	if query.Id != 0 {
+		sql.WriteString(` AND alert_notification.id = ?`)
+		params = append(params, strconv.Itoa(int(query.Id)))
+	}
+
+	if len(query.Ids) > 0 {
+		sql.WriteString(` AND (`)
+
+		for i, id := range query.Ids {
+			if i != 0 {
+				sql.WriteString(` OR`)
+			}
+			sql.WriteString(` alert_notification.id = ?`)
+			params = append(params, id)
+		}
+
+		sql.WriteString(`)`)
+	}
+
+	var searches []*m.AlertNotification
+	if err := sess.Sql(sql.String(), params...).Find(&searches); err != nil {
+		return err
+	}
+
+	var result []*m.AlertNotification
+	var def []*m.AlertNotification
+	if query.IncludeAlwaysExecute {
+
+		if err := sess.Where("org_id = ? AND always_execute = 1", query.OrgID).Find(&def); err != nil {
+			return err
+		}
+
+		result = append(result, def...)
+	}
+
+	for _, s := range searches {
+		canAppend := true
+		for _, d := range result {
+			if d.Id == s.Id {
+				canAppend = false
+				break
+			}
+		}
+
+		if canAppend {
+			result = append(result, s)
+		}
+	}
+
+	query.Result = result
+	return nil
+}
+
+func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error {
+	return inTransaction(func(sess *xorm.Session) error {
+		existingQuery := &m.GetAlertNotificationQuery{OrgID: cmd.OrgID, Name: cmd.Name, IncludeAlwaysExecute: false}
+		err := getAlertNotifications(existingQuery, sess)
+
+		if err != nil {
+			return err
+		}
+
+		if len(existingQuery.Result) > 0 {
+			return fmt.Errorf("Alert notification name %s already exists", cmd.Name)
+		}
+
+		alertNotification := &m.AlertNotification{
+			OrgId:         cmd.OrgID,
+			Name:          cmd.Name,
+			Type:          cmd.Type,
+			Created:       time.Now(),
+			Settings:      cmd.Settings,
+			Updated:       time.Now(),
+			AlwaysExecute: cmd.AlwaysExecute,
+		}
+
+		_, err = sess.Insert(alertNotification)
+
+		if err != nil {
+			return err
+		}
+
+		cmd.Result = alertNotification
+		return nil
+	})
+}
+
+func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
+	return inTransaction(func(sess *xorm.Session) (err error) {
+		current := &m.AlertNotification{}
+		_, err = sess.Id(cmd.Id).Get(current)
+
+		if err != nil {
+			return err
+		}
+
+		alertNotification := &m.AlertNotification{
+			Id:            cmd.Id,
+			OrgId:         cmd.OrgID,
+			Name:          cmd.Name,
+			Type:          cmd.Type,
+			Settings:      cmd.Settings,
+			Updated:       time.Now(),
+			Created:       current.Created,
+			AlwaysExecute: cmd.AlwaysExecute,
+		}
+
+		sess.UseBool("always_execute")
+
+		var affected int64
+		affected, err = sess.Id(alertNotification.Id).Update(alertNotification)
+
+		if err != nil {
+			return err
+		}
+
+		if affected == 0 {
+			return fmt.Errorf("Could not find alert notification")
+		}
+
+		cmd.Result = alertNotification
+		return nil
+	})
+}

pkg/services/sqlstore/alert_notification_test.go

@@ -0,0 +1,112 @@
+package sqlstore
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAlertNotificationSQLAccess(t *testing.T) {
+	Convey("Testing Alert notification sql access", t, func() {
+		InitTestDB(t)
+		var err error
+
+		Convey("Alert notifications should be empty", func() {
+			cmd := &m.GetAlertNotificationQuery{
+				OrgID: FakeOrgId,
+				Name:  "email",
+			}
+
+			err := AlertNotificationQuery(cmd)
+			fmt.Printf("errror %v", err)
+			So(err, ShouldBeNil)
+			So(len(cmd.Result), ShouldEqual, 0)
+		})
+
+		Convey("Can save Alert Notification", func() {
+			cmd := &m.CreateAlertNotificationCommand{
+				Name:          "ops",
+				Type:          "email",
+				OrgID:         1,
+				Settings:      simplejson.New(),
+				AlwaysExecute: true,
+			}
+
+			err = CreateAlertNotificationCommand(cmd)
+			So(err, ShouldBeNil)
+			So(cmd.Result.Id, ShouldNotEqual, 0)
+			So(cmd.Result.OrgId, ShouldNotEqual, 0)
+			So(cmd.Result.Type, ShouldEqual, "email")
+			So(cmd.Result.AlwaysExecute, ShouldEqual, true)
+
+			Convey("Cannot save Alert Notification with the same name", func() {
+				err = CreateAlertNotificationCommand(cmd)
+				So(err, ShouldNotBeNil)
+			})
+
+			Convey("Can update alert notification", func() {
+				newCmd := &m.UpdateAlertNotificationCommand{
+					Name:          "NewName",
+					Type:          "webhook",
+					OrgID:         cmd.Result.OrgId,
+					Settings:      simplejson.New(),
+					Id:            cmd.Result.Id,
+					AlwaysExecute: true,
+				}
+				err := UpdateAlertNotification(newCmd)
+				So(err, ShouldBeNil)
+				So(newCmd.Result.Name, ShouldEqual, "NewName")
+			})
+		})
+
+		Convey("Can search using an array of ids", func() {
+			So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{
+				Name:          "nagios",
+				Type:          "webhook",
+				OrgID:         1,
+				Settings:      simplejson.New(),
+				AlwaysExecute: true,
+			}), ShouldBeNil)
+
+			So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{
+				Name:     "ops2",
+				Type:     "email",
+				OrgID:    1,
+				Settings: simplejson.New(),
+			}), ShouldBeNil)
+
+			So(CreateAlertNotificationCommand(&m.CreateAlertNotificationCommand{
+				Name:     "slack",
+				Type:     "webhook",
+				OrgID:    1,
+				Settings: simplejson.New(),
+			}), ShouldBeNil)
+
+			Convey("search", func() {
+				existingNotification := int64(2)
+				missingThatSholdNotCauseerrors := int64(99)
+
+				query := &m.GetAlertNotificationQuery{
+					Ids:                  []int64{existingNotification, missingThatSholdNotCauseerrors},
+					OrgID:                1,
+					IncludeAlwaysExecute: true,
+				}
+
+				err := AlertNotificationQuery(query)
+				So(err, ShouldBeNil)
+				So(len(query.Result), ShouldEqual, 2)
+				defaultNotifications := 0
+				for _, not := range query.Result {
+					if not.AlwaysExecute {
+						defaultNotifications++
+					}
+				}
+
+				So(defaultNotifications, ShouldEqual, 1)
+			})
+		})
+	})
+}

pkg/services/sqlstore/alert_state.go

@@ -12,6 +12,23 @@ import (
 func init() {
 	bus.AddHandler("sql", SetNewAlertState)
 	bus.AddHandler("sql", GetAlertStateLogByAlertId)
+	bus.AddHandler("sql", GetLastAlertStateQuery)
+}
+
+func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
+	states := make([]m.AlertState, 0)
+
+	if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
+		return err
+	}
+
+	if len(states) == 0 {
+		cmd.Result = nil
+		return fmt.Errorf("invalid amount of alertstates. Expected 1 got %v", len(states))
+	}
+
+	cmd.Result = &states[0]
+	return nil
 }
 
 func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
@@ -30,20 +47,16 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
 			return fmt.Errorf("Could not find alert")
 		}
 
-		if alert.State == cmd.NewState {
-			cmd.Result = &m.Alert{}
-			return nil
-		}
-
 		alert.State = cmd.NewState
 		sess.Id(alert.Id).Update(&alert)
 
 		alertState := m.AlertState{
-			AlertId:  cmd.AlertId,
-			OrgId:    cmd.AlertId,
-			NewState: cmd.NewState,
-			Info:     cmd.Info,
-			Created:  time.Now(),
+			AlertId:         cmd.AlertId,
+			OrgId:           cmd.OrgId,
+			NewState:        cmd.NewState,
+			Info:            cmd.Info,
+			Created:         time.Now(),
+			TriggeredAlerts: cmd.TriggeredAlerts,
 		}
 
 		sess.Insert(&alertState)
@@ -54,12 +67,12 @@ func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
 }
 
 func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
-	alertLogs := make([]m.AlertState, 0)
+	states := make([]m.AlertState, 0)
 
-	if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&alertLogs); err != nil {
+	if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
 		return err
 	}
 
-	cmd.Result = &alertLogs
+	cmd.Result = &states
 	return nil
 }

pkg/services/sqlstore/dashboard.go

@@ -249,10 +249,10 @@ func GetDashboards(query *m.GetDashboardsQuery) error {
 		return m.ErrCommandValidationFailed
 	}
 
-	var dashboards = make([]m.Dashboard, 0)
+	var dashboards = make([]*m.Dashboard, 0)
 
 	err := x.In("id", query.DashboardIds).Find(&dashboards)
-	query.Result = &dashboards
+	query.Result = dashboards
 
 	if err != nil {
 		return err

pkg/services/sqlstore/migrations/alert_mig.go

@@ -49,6 +49,7 @@ func addAlertMigrations(mg *Migrator) {
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
 			{Name: "new_state", Type: DB_NVarchar, Length: 50, Nullable: false},
 			{Name: "info", Type: DB_Text, Nullable: true},
+			{Name: "triggered_alerts", Type: DB_Text, Nullable: true},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 		},
 	}
@@ -66,4 +67,20 @@ func addAlertMigrations(mg *Migrator) {
 	}
 
 	mg.AddMigration("create alert_heartbeat table v1", NewAddTableMigration(alert_heartbeat))
+
+	alert_notification := Table{
+		Name: "alert_notification",
+		Columns: []*Column{
+			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "org_id", Type: DB_BigInt, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "always_execute", Type: DB_Bool, Nullable: false},
+			{Name: "settings", Type: DB_Text, Nullable: false},
+			{Name: "created", Type: DB_DateTime, Nullable: false},
+			{Name: "updated", Type: DB_DateTime, Nullable: false},
+		},
+	}
+
+	mg.AddMigration("create alert_notification table v1", NewAddTableMigration(alert_notification))
 }