From ccfd9c89b2645fde4b12aad0819c178ef5afb979 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 1 Nov 2018 16:04:38 +0100
Subject: [PATCH 01/23] introduces hard coded deboucing for alerting

---
 pkg/services/alerting/eval_context.go      |  21 ++-
 pkg/services/alerting/eval_context_test.go | 186 +++++++++++++--------
 pkg/services/alerting/result_handler.go    |   3 +
 pkg/services/alerting/rule.go              |   5 +
 4 files changed, 145 insertions(+), 70 deletions(-)

diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index d0441d379b7..49e28bbf5ec 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -69,7 +69,7 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 			Text:  "Alerting",
 		}
 	default:
-		panic("Unknown rule state " + c.Rule.State)
+		panic("Unknown rule state for alert notifications " + c.Rule.State)
 	}
 }
 
@@ -125,11 +125,26 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 			return c.PrevAlertState
 		}
 		return c.Rule.ExecutionErrorState.ToAlertState()
+	}
 
-	} else if c.Firing {
+	if c.Firing && c.Rule.DebounceDuration != 0 {
+		since := time.Now().Sub(c.Rule.LastStateChange)
+		if since > c.Rule.DebounceDuration {
+			return m.AlertStateAlerting
+		}
+
+		if c.PrevAlertState == m.AlertStateAlerting {
+			return m.AlertStateAlerting
+		}
+
+		return m.AlertStatePending
+	}
+
+	if c.Firing {
 		return m.AlertStateAlerting
+	}
 
-	} else if c.NoDataFound {
+	if c.NoDataFound {
 		c.log.Info("Alert Rule returned no data",
 			"ruleId", c.Rule.Id,
 			"name", c.Rule.Name,
diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go
index 750fa959683..2abf581d830 100644
--- a/pkg/services/alerting/eval_context_test.go
+++ b/pkg/services/alerting/eval_context_test.go
@@ -2,11 +2,11 @@ package alerting
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/models"
-	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestStateIsUpdatedWhenNeeded(t *testing.T) {
@@ -31,71 +31,123 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
 	})
 }
 
-func TestAlertingEvalContext(t *testing.T) {
-	Convey("Should compute and replace properly new rule state", t, func() {
+func TestGetStateFromEvalContext(t *testing.T) {
+	tcs := []struct {
+		name     string
+		expected models.AlertStateType
+		applyFn  func(ec *EvalContext)
+		focus    bool
+	}{
+		{
+			name:     "ok -> alerting",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.Firing = true
+				ec.PrevAlertState = models.AlertStateOK
+			},
+		},
+		{
+			name:     "ok -> error(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+			},
+		},
+		{
+			name:     "ok -> pending. since its been firing for less than FOR",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+				ec.Rule.DebounceDuration = time.Minute * 5
+			},
+		},
+		{
+			name:     "ok -> alerting. since its been firing for more than FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.DebounceDuration = time.Minute * 2
+			},
+		},
+		{
+			name:     "alerting -> alerting. should not update regardless of FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateAlerting
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.DebounceDuration = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> ok. should not update regardless of FOR",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.DebounceDuration = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> error(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "pending -> error(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "ok -> no_data(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "ok -> no_data(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "pending -> no_data(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+	}
+
+	for _, tc := range tcs {
 		ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
-		dummieError := fmt.Errorf("dummie error")
 
-		Convey("ok -> alerting", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Firing = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-
-		Convey("ok -> no_data(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataSetAlerting
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-	})
+		tc.applyFn(ctx)
+		have := ctx.GetNewState()
+		if have != tc.expected {
+			t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
+		}
+	}
 }
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
index 420ffeb9a55..ce12a8a6b96 100644
--- a/pkg/services/alerting/result_handler.go
+++ b/pkg/services/alerting/result_handler.go
@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			// when two servers are raising. This makes sure that the server
 			// with the last state change always sends a notification.
 			evalContext.Rule.StateChanges = cmd.Result.StateChanges
+
+			// Update the last state change of the alert rule in memory
+			evalContext.Rule.LastStateChange = time.Now()
 		}
 
 		// save annotation
diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index 999611f15c4..3fb69b48f9f 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 
@@ -18,6 +19,8 @@ type Rule struct {
 	Frequency           int64
 	Name                string
 	Message             string
+	LastStateChange     time.Time
+	DebounceDuration    time.Duration
 	NoDataState         m.NoDataOption
 	ExecutionErrorState m.ExecutionErrorOption
 	State               m.AlertStateType
@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
+	model.LastStateChange = ruleDef.NewStateDate
+	model.DebounceDuration = time.Minute * 2 // hard coded for now
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges

From 2d3a5754891ddc093524dc72138272a72160694f Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 2 Nov 2018 09:00:56 +0100
Subject: [PATCH 02/23] adds db migration for debounce_duration

---
 pkg/models/alert.go                           | 27 ++++++++++---------
 pkg/services/alerting/rule.go                 |  2 +-
 pkg/services/sqlstore/migrations/alert_mig.go |  4 +++
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/pkg/models/alert.go b/pkg/models/alert.go
index ba1fc0779ba..e35ba106688 100644
--- a/pkg/models/alert.go
+++ b/pkg/models/alert.go
@@ -59,19 +59,20 @@ func (s ExecutionErrorOption) ToAlertState() AlertStateType {
 }
 
 type Alert struct {
-	Id             int64
-	Version        int64
-	OrgId          int64
-	DashboardId    int64
-	PanelId        int64
-	Name           string
-	Message        string
-	Severity       string
-	State          AlertStateType
-	Handler        int64
-	Silenced       bool
-	ExecutionError string
-	Frequency      int64
+	Id               int64
+	Version          int64
+	OrgId            int64
+	DashboardId      int64
+	PanelId          int64
+	Name             string
+	Message          string
+	Severity         string //Unused
+	State            AlertStateType
+	Handler          int64 //Unused
+	Silenced         bool
+	ExecutionError   string
+	Frequency        int64
+	DebounceDuration time.Duration
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time
diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index 3fb69b48f9f..c9fbddbf393 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -104,7 +104,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
 	model.LastStateChange = ruleDef.NewStateDate
-	model.DebounceDuration = time.Minute * 2 // hard coded for now
+	model.DebounceDuration = time.Duration(ruleDef.DebounceDuration)
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index 198a47b50ff..f575fb3b02b 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
 	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
 		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
+
+	mg.AddMigration("Add decounce_duration to alert table", NewAddColumnMigration(alertV1, &Column{
+		Name: "debounce_duration", Type: DB_BigInt, Nullable: true,
+	}))
 }

From 4526660cb2ad1e5a1b555cf15b4c89c4e471ad6d Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 2 Nov 2018 10:38:02 +0100
Subject: [PATCH 03/23] wire up debounce setting in the ui

---
 pkg/services/alerting/extractor.go            |  27 +-
 pkg/services/sqlstore/alert.go                |   3 +-
 public/app/features/alerting/AlertTabCtrl.ts  |   1 +
 .../features/alerting/partials/alert_tab.html | 272 +++++++++---------
 4 files changed, 164 insertions(+), 139 deletions(-)

diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go
index edfab2dedee..221d58feaf2 100644
--- a/pkg/services/alerting/extractor.go
+++ b/pkg/services/alerting/extractor.go
@@ -2,6 +2,7 @@ package alerting
 
 import (
 	"errors"
+	"time"
 
 	"fmt"
 
@@ -113,15 +114,25 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			return nil, ValidationError{Reason: "Could not parse frequency"}
 		}
 
+		rawDebouce := jsonAlert.Get("debounceDuration").MustString()
+		var debounceDuration time.Duration
+		if rawDebouce != "" {
+			debounceDuration, err = time.ParseDuration(rawDebouce)
+			if err != nil {
+				return nil, ValidationError{Reason: "Could not parse debounceDuration"}
+			}
+		}
+
 		alert := &m.Alert{
-			DashboardId: e.Dash.Id,
-			OrgId:       e.OrgID,
-			PanelId:     panelID,
-			Id:          jsonAlert.Get("id").MustInt64(),
-			Name:        jsonAlert.Get("name").MustString(),
-			Handler:     jsonAlert.Get("handler").MustInt64(),
-			Message:     jsonAlert.Get("message").MustString(),
-			Frequency:   frequency,
+			DashboardId:      e.Dash.Id,
+			OrgId:            e.OrgID,
+			PanelId:          panelID,
+			Id:               jsonAlert.Get("id").MustInt64(),
+			Name:             jsonAlert.Get("name").MustString(),
+			Handler:          jsonAlert.Get("handler").MustInt64(),
+			Message:          jsonAlert.Get("message").MustString(),
+			Frequency:        frequency,
+			DebounceDuration: debounceDuration,
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {
diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go
index 2f17402b80c..88ffa3a9b4e 100644
--- a/pkg/services/sqlstore/alert.go
+++ b/pkg/services/sqlstore/alert.go
@@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message")
+				sess.MustCols("message", "debounce_duration")
+
 				_, err := sess.ID(alert.Id).Update(alert)
 				if err != nil {
 					return err
diff --git a/public/app/features/alerting/AlertTabCtrl.ts b/public/app/features/alerting/AlertTabCtrl.ts
index 146b7026353..43752d0a8c9 100644
--- a/public/app/features/alerting/AlertTabCtrl.ts
+++ b/public/app/features/alerting/AlertTabCtrl.ts
@@ -169,6 +169,7 @@ export class AlertTabCtrl {
     alert.frequency = alert.frequency || '1m';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
+    alert.debounceDuration = alert.debounceDuration || '5m';
 
     const defaultName = this.panel.title + ' alert';
     alert.name = alert.name || defaultName;
diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html
index cb101672aa4..de5fc4df382 100644
--- a/public/app/features/alerting/partials/alert_tab.html
+++ b/public/app/features/alerting/partials/alert_tab.html
@@ -1,147 +1,159 @@
 <div class="edit-tab-with-sidemenu" ng-if="ctrl.alert">
-	<aside class="edit-sidemenu-aside">
-		<ul class="edit-sidemenu">
-			<li ng-class="{active: ctrl.subTabIndex === 0}">
-				<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
-			</li>
-			<li ng-class="{active: ctrl.subTabIndex === 1}">
-				<a ng-click="ctrl.changeTabIndex(1)">
-					Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
-				</a>
-			</li>
-			<li ng-class="{active: ctrl.subTabIndex === 2}">
-				<a ng-click="ctrl.changeTabIndex(2)">State history</a>
-			</li>
+  <aside class="edit-sidemenu-aside">
+    <ul class="edit-sidemenu">
+      <li ng-class="{active: ctrl.subTabIndex === 0}">
+        <a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
+      </li>
+      <li ng-class="{active: ctrl.subTabIndex === 1}">
+        <a ng-click="ctrl.changeTabIndex(1)">
+          Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
+        </a>
+      </li>
+      <li ng-class="{active: ctrl.subTabIndex === 2}">
+        <a ng-click="ctrl.changeTabIndex(2)">State history</a>
+      </li>
       <li>
-				<a ng-click="ctrl.delete()">Delete</a>
-			</li>
-		</ul>
-	</aside>
+        <a ng-click="ctrl.delete()">Delete</a>
+      </li>
+    </ul>
+  </aside>
 
-	<div class="edit-tab-content">
-		<div ng-if="ctrl.subTabIndex === 0">
-			<div class="alert alert-error m-b-2" ng-show="ctrl.error">
-				<i class="fa fa-warning"></i> {{ctrl.error}}
-			</div>
+  <div class="edit-tab-content">
+    <div ng-if="ctrl.subTabIndex === 0">
+      <div class="alert alert-error m-b-2" ng-show="ctrl.error">
+        <i class="fa fa-warning"></i> {{ctrl.error}}
+      </div>
 
-			<div class="gf-form-group">
-				<h5 class="section-heading">Alert Config</h5>
-				<div class="gf-form">
-					<span class="gf-form-label width-6">Name</span>
-					<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
-					<span class="gf-form-label">Evaluate every</span>
-					<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
-				</div>
-			</div>
+      <div class="gf-form-group">
+        <h5 class="section-heading">Alert Config</h5>
+        <div class="gf-form">
+          <span class="gf-form-label width-6">Name</span>
+          <input type="text" class="gf-form-input width-22" ng-model="ctrl.alert.name">
+        </div>
+        <div class="gf-form-inline">
+          <div class="gf-form">
+            <span class="gf-form-label width-8">Evaluate every</span>
+            <input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency">
+          </div>
+          <div class="gf-form max-width-15">
+            <label class="gf-form-label width-10">Debounce duration</label>
+            <input type="text" class="gf-form-input max-width-5" ng-model="ctrl.alert.debounceDuration" spellcheck='false' placeholder="5m">
+            <info-popover mode="right-absolute">
+              Configuring this value means that an alert rule have to be firing for atleast this duration before changing state.
+              This should reduce false positive alerts and avoid flapping alerts.
+            </info-popover>
+          </div>
+        </div>
+      </div>
 
-			<div class="gf-form-group">
-				<h5 class="section-heading">Conditions</h5>
-				<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
-					<div class="gf-form">
-						<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
-						<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
-					</div>
-          			<div class="gf-form">
-						<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
-						</query-part-editor>
-            			<span class="gf-form-label query-keyword">OF</span>
-					</div>
-					<div class="gf-form">
-						<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
-						</query-part-editor>
-					</div>
-					<div class="gf-form">
-						<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
-						<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
-            			<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
-            			<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
-					</div>
-					<div class="gf-form">
-						<label class="gf-form-label">
-							<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
-								<i class="fa fa-trash"></i>
-							</a>
-						</label>
-					</div>
-				</div>
+      <div class="gf-form-group">
+        <h5 class="section-heading">Conditions</h5>
+        <div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
+          <div class="gf-form">
+            <metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
+            <span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
+          </div>
+                <div class="gf-form">
+            <query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
+            </query-part-editor>
+                  <span class="gf-form-label query-keyword">OF</span>
+          </div>
+          <div class="gf-form">
+            <query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
+            </query-part-editor>
+          </div>
+          <div class="gf-form">
+            <metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
+            <input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
+                  <label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
+                  <input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
+          </div>
+          <div class="gf-form">
+            <label class="gf-form-label">
+              <a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
+                <i class="fa fa-trash"></i>
+              </a>
+            </label>
+          </div>
+        </div>
 
-				<div class="gf-form">
-					<label class="gf-form-label dropdown">
-						<a class="pointer dropdown-toggle" data-toggle="dropdown">
-							<i class="fa fa-plus"></i>
-						</a>
-						<ul class="dropdown-menu" role="menu">
-							<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
-								<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
-							</li>
-						</ul>
-					</label>
-				</div>
-			</div>
+        <div class="gf-form">
+          <label class="gf-form-label dropdown">
+            <a class="pointer dropdown-toggle" data-toggle="dropdown">
+              <i class="fa fa-plus"></i>
+            </a>
+            <ul class="dropdown-menu" role="menu">
+              <li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
+                <a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
+              </li>
+            </ul>
+          </label>
+        </div>
+      </div>
 
-			<div class="gf-form-group">
-				<div class="gf-form">
-          			<span class="gf-form-label width-18">If no data or all values are null</span>
-          			<span class="gf-form-label query-keyword">SET STATE TO</span>
-					<div class="gf-form-select-wrapper">
-						<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
-						</select>
-					</div>
-				</div>
+      <div class="gf-form-group">
+        <div class="gf-form">
+                <span class="gf-form-label width-18">If no data or all values are null</span>
+                <span class="gf-form-label query-keyword">SET STATE TO</span>
+          <div class="gf-form-select-wrapper">
+            <select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
+            </select>
+          </div>
+        </div>
 
-				<div class="gf-form">
-          			<span class="gf-form-label width-18">If execution error or timeout</span>
-          			<span class="gf-form-label query-keyword">SET STATE TO</span>
-					<div class="gf-form-select-wrapper">
-						<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
-						</select>
-					</div>
-				</div>
+        <div class="gf-form">
+                <span class="gf-form-label width-18">If execution error or timeout</span>
+                <span class="gf-form-label query-keyword">SET STATE TO</span>
+          <div class="gf-form-select-wrapper">
+            <select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
+            </select>
+          </div>
+        </div>
 
-				<div class="gf-form-button-row">
-					<button class="btn btn-inverse" ng-click="ctrl.test()">
-						Test Rule
-					</button>
-				</div>
-			</div>
+        <div class="gf-form-button-row">
+          <button class="btn btn-inverse" ng-click="ctrl.test()">
+            Test Rule
+          </button>
+        </div>
+      </div>
 
-			<div class="gf-form-group" ng-if="ctrl.testing">
-				Evaluating rule <i class="fa fa-spinner fa-spin"></i>
-			</div>
+      <div class="gf-form-group" ng-if="ctrl.testing">
+        Evaluating rule <i class="fa fa-spinner fa-spin"></i>
+      </div>
 
-			<div class="gf-form-group" ng-if="ctrl.testResult">
-				<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
-			</div>
-		</div>
+      <div class="gf-form-group" ng-if="ctrl.testResult">
+        <json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
+      </div>
+    </div>
 
-		<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
-			<h5 class="section-heading">Notifications</h5>
-			<div class="gf-form-inline">
-				<div class="gf-form max-width-30">
-					<span class="gf-form-label width-8">Send to</span>
-					<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
-						<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
-						<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
-					</span>
-					<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
-				</div>
-			</div>
-			<div class="gf-form gf-form--v-stretch">
-				<span class="gf-form-label width-8">Message</span>
-				<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message"  placeholder="Notification message details..."></textarea>
-			</div>
-		</div>
+    <div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
+      <h5 class="section-heading">Notifications</h5>
+      <div class="gf-form-inline">
+        <div class="gf-form max-width-30">
+          <span class="gf-form-label width-8">Send to</span>
+          <span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
+            <i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
+            <i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
+          </span>
+          <metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
+        </div>
+      </div>
+      <div class="gf-form gf-form--v-stretch">
+        <span class="gf-form-label width-8">Message</span>
+        <textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message"  placeholder="Notification message details..."></textarea>
+      </div>
+    </div>
 
-		<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
-			<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
-      		<h5 class="section-heading" style="whitespace: nowrap">
-				State history <span class="muted small">(last 50 state changes)</span>
-			</h5>
+    <div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
+      <button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
+          <h5 class="section-heading" style="whitespace: nowrap">
+        State history <span class="muted small">(last 50 state changes)</span>
+      </h5>
 
-			<div ng-show="ctrl.alertHistory.length === 0">
-				<br>
-				<i>No state changes recorded</i>
-			</div>
+      <div ng-show="ctrl.alertHistory.length === 0">
+        <br>
+        <i>No state changes recorded</i>
+      </div>
 
       <ol class="alert-rule-list" >
         <li class="alert-rule-item" ng-repeat="al in ctrl.alertHistory">

From 6f748d8a96ee5a1ee21fb0f9c9309724086252eb Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 2 Nov 2018 13:53:47 +0100
Subject: [PATCH 04/23] fixes go meta lint issue

---
 pkg/services/alerting/rule.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index c9fbddbf393..ac1885fb380 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -104,7 +104,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
 	model.LastStateChange = ruleDef.NewStateDate
-	model.DebounceDuration = time.Duration(ruleDef.DebounceDuration)
+	model.DebounceDuration = ruleDef.DebounceDuration
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges

From d25284a36441ef3ec2f1e8953acbd7fb4680f7a9 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 5 Nov 2018 10:23:43 +0100
Subject: [PATCH 05/23] introduce state `unknown` for rules that have not been
 evaluated yet

---
 pkg/api/alerting.go                            |  2 +-
 pkg/models/alert.go                            |  8 +++++++-
 pkg/services/alerting/eval_context.go          |  5 +++++
 pkg/services/alerting/notifiers/base.go        |  5 +++++
 pkg/services/alerting/notifiers/base_test.go   | 16 ++++++++++++++++
 pkg/services/sqlstore/alert.go                 |  6 +++---
 pkg/services/sqlstore/alert_test.go            |  4 ++--
 public/app/features/alerting/state/alertDef.ts |  7 +++++++
 8 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go
index a936d696207..b007b3a3492 100644
--- a/pkg/api/alerting.go
+++ b/pkg/api/alerting.go
@@ -291,7 +291,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response {
 		return Error(500, "", err)
 	}
 
-	var response m.AlertStateType = m.AlertStatePending
+	var response m.AlertStateType = m.AlertStateUnknown
 	pausedState := "un-paused"
 	if cmd.Paused {
 		response = m.AlertStatePaused
diff --git a/pkg/models/alert.go b/pkg/models/alert.go
index e35ba106688..37f40134796 100644
--- a/pkg/models/alert.go
+++ b/pkg/models/alert.go
@@ -19,6 +19,7 @@ const (
 	AlertStateAlerting AlertStateType = "alerting"
 	AlertStateOK       AlertStateType = "ok"
 	AlertStatePending  AlertStateType = "pending"
+	AlertStateUnknown  AlertStateType = "unknown"
 )
 
 const (
@@ -39,7 +40,12 @@ var (
 )
 
 func (s AlertStateType) IsValid() bool {
-	return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
+	return s == AlertStateOK ||
+		s == AlertStateNoData ||
+		s == AlertStatePaused ||
+		s == AlertStatePending ||
+		s == AlertStateAlerting ||
+		s == AlertStateUnknown
 }
 
 func (s NoDataOption) IsValid() bool {
diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 49e28bbf5ec..8986af85406 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -68,6 +68,11 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 			Color: "#D63232",
 			Text:  "Alerting",
 		}
+	case m.AlertStateUnknown:
+		return &StateDescription{
+			Color: "888888",
+			Text:  "Unknown",
+		}
 	default:
 		panic("Unknown rule state for alert notifications " + c.Rule.State)
 	}
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index d141d6cd257..35d3ff518a0 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -67,6 +67,11 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
 	}
 
 	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
+		return false
+	}
+
+	// Do not notify when we become OK from pending
 	if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
 		return false
 	}
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 5062828cb4f..388c2db17ee 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -132,6 +132,22 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState: m.AlertStateOK,
 			state:     &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
 
+			expect: true,
+		},
+		{
+			name:      "unknown -> ok",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateOK,
+			state:     &m.AlertNotificationState{},
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> alerting",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateAlerting,
+			state:     &m.AlertNotificationState{},
+
 			expect: true,
 		},
 	}
diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go
index 88ffa3a9b4e..78a71cc8497 100644
--- a/pkg/services/sqlstore/alert.go
+++ b/pkg/services/sqlstore/alert.go
@@ -205,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 		} else {
 			alert.Updated = timeNow()
 			alert.Created = timeNow()
-			alert.State = m.AlertStatePending
+			alert.State = m.AlertStateUnknown
 			alert.NewStateDate = timeNow()
 
 			_, err := sess.Insert(alert)
@@ -300,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error {
 			params = append(params, string(m.AlertStatePaused))
 			params = append(params, timeNow())
 		} else {
-			params = append(params, string(m.AlertStatePending))
+			params = append(params, string(m.AlertStateUnknown))
 			params = append(params, timeNow())
 		}
 
@@ -324,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error {
 		if cmd.Paused {
 			newState = string(m.AlertStatePaused)
 		} else {
-			newState = string(m.AlertStatePending)
+			newState = string(m.AlertStateUnknown)
 		}
 
 		res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow())
diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go
index d97deb45f0e..40867e96b4d 100644
--- a/pkg/services/sqlstore/alert_test.go
+++ b/pkg/services/sqlstore/alert_test.go
@@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) {
 			So(alert.DashboardId, ShouldEqual, testDash.Id)
 			So(alert.PanelId, ShouldEqual, 1)
 			So(alert.Name, ShouldEqual, "Alerting title")
-			So(alert.State, ShouldEqual, "pending")
+			So(alert.State, ShouldEqual, m.AlertStateUnknown)
 			So(alert.NewStateDate, ShouldNotBeNil)
 			So(alert.EvalData, ShouldNotBeNil)
 			So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test")
@@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) {
 				So(query.Result[0].Name, ShouldEqual, "Name")
 
 				Convey("Alert state should not be updated", func() {
-					So(query.Result[0].State, ShouldEqual, "pending")
+					So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown)
 				})
 			})
 
diff --git a/public/app/features/alerting/state/alertDef.ts b/public/app/features/alerting/state/alertDef.ts
index 11d2aafaa7f..378be0afb91 100644
--- a/public/app/features/alerting/state/alertDef.ts
+++ b/public/app/features/alerting/state/alertDef.ts
@@ -99,6 +99,13 @@ function getStateDisplayModel(state) {
         stateClass: 'alert-state-warning',
       };
     }
+    case 'unknown': {
+      return {
+        text: 'UNKNOWN',
+        iconClass: 'fa fa-question',
+        stateClass: 'alert-state-paused',
+      };
+    }
   }
 
   throw { message: 'Unknown alert state' };

From ccd89eee974d08f7f74f7859767e8f32a347e2a5 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 5 Nov 2018 11:05:30 +0100
Subject: [PATCH 06/23] renames `debouceduration` to `for`

---
 pkg/models/alert.go                           | 28 +++++++++----------
 pkg/services/alerting/eval_context.go         |  4 +--
 pkg/services/alerting/eval_context_test.go    |  8 +++---
 pkg/services/alerting/extractor.go            | 28 +++++++++----------
 pkg/services/alerting/rule.go                 |  4 +--
 pkg/services/sqlstore/migrations/alert_mig.go |  4 +--
 public/app/features/alerting/AlertTabCtrl.ts  |  2 +-
 .../features/alerting/partials/alert_tab.html | 12 ++++----
 8 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/pkg/models/alert.go b/pkg/models/alert.go
index 37f40134796..760e9eada48 100644
--- a/pkg/models/alert.go
+++ b/pkg/models/alert.go
@@ -65,20 +65,20 @@ func (s ExecutionErrorOption) ToAlertState() AlertStateType {
 }
 
 type Alert struct {
-	Id               int64
-	Version          int64
-	OrgId            int64
-	DashboardId      int64
-	PanelId          int64
-	Name             string
-	Message          string
-	Severity         string //Unused
-	State            AlertStateType
-	Handler          int64 //Unused
-	Silenced         bool
-	ExecutionError   string
-	Frequency        int64
-	DebounceDuration time.Duration
+	Id             int64
+	Version        int64
+	OrgId          int64
+	DashboardId    int64
+	PanelId        int64
+	Name           string
+	Message        string
+	Severity       string //Unused
+	State          AlertStateType
+	Handler        int64 //Unused
+	Silenced       bool
+	ExecutionError string
+	Frequency      int64
+	For            time.Duration
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time
diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 8986af85406..208fe1d188b 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -132,9 +132,9 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 		return c.Rule.ExecutionErrorState.ToAlertState()
 	}
 
-	if c.Firing && c.Rule.DebounceDuration != 0 {
+	if c.Firing && c.Rule.For != 0 {
 		since := time.Now().Sub(c.Rule.LastStateChange)
-		if since > c.Rule.DebounceDuration {
+		if since > c.Rule.For {
 			return m.AlertStateAlerting
 		}
 
diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go
index 2abf581d830..cc0bed79d10 100644
--- a/pkg/services/alerting/eval_context_test.go
+++ b/pkg/services/alerting/eval_context_test.go
@@ -62,7 +62,7 @@ func TestGetStateFromEvalContext(t *testing.T) {
 				ec.PrevAlertState = models.AlertStateOK
 				ec.Firing = true
 				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
-				ec.Rule.DebounceDuration = time.Minute * 5
+				ec.Rule.For = time.Minute * 5
 			},
 		},
 		{
@@ -72,7 +72,7 @@ func TestGetStateFromEvalContext(t *testing.T) {
 				ec.PrevAlertState = models.AlertStateOK
 				ec.Firing = true
 				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
-				ec.Rule.DebounceDuration = time.Minute * 2
+				ec.Rule.For = time.Minute * 2
 			},
 		},
 		{
@@ -82,7 +82,7 @@ func TestGetStateFromEvalContext(t *testing.T) {
 				ec.PrevAlertState = models.AlertStateAlerting
 				ec.Firing = true
 				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
-				ec.Rule.DebounceDuration = time.Minute * 2
+				ec.Rule.For = time.Minute * 2
 			},
 		},
 		{
@@ -91,7 +91,7 @@ func TestGetStateFromEvalContext(t *testing.T) {
 			applyFn: func(ec *EvalContext) {
 				ec.PrevAlertState = models.AlertStateOK
 				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
-				ec.Rule.DebounceDuration = time.Minute * 2
+				ec.Rule.For = time.Minute * 2
 			},
 		},
 		{
diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go
index 221d58feaf2..244dc0a0770 100644
--- a/pkg/services/alerting/extractor.go
+++ b/pkg/services/alerting/extractor.go
@@ -114,25 +114,25 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			return nil, ValidationError{Reason: "Could not parse frequency"}
 		}
 
-		rawDebouce := jsonAlert.Get("debounceDuration").MustString()
-		var debounceDuration time.Duration
-		if rawDebouce != "" {
-			debounceDuration, err = time.ParseDuration(rawDebouce)
+		rawFow := jsonAlert.Get("for").MustString()
+		var forValue time.Duration
+		if rawFow != "" {
+			forValue, err = time.ParseDuration(rawFow)
 			if err != nil {
-				return nil, ValidationError{Reason: "Could not parse debounceDuration"}
+				return nil, ValidationError{Reason: "Could not parse for"}
 			}
 		}
 
 		alert := &m.Alert{
-			DashboardId:      e.Dash.Id,
-			OrgId:            e.OrgID,
-			PanelId:          panelID,
-			Id:               jsonAlert.Get("id").MustInt64(),
-			Name:             jsonAlert.Get("name").MustString(),
-			Handler:          jsonAlert.Get("handler").MustInt64(),
-			Message:          jsonAlert.Get("message").MustString(),
-			Frequency:        frequency,
-			DebounceDuration: debounceDuration,
+			DashboardId: e.Dash.Id,
+			OrgId:       e.OrgID,
+			PanelId:     panelID,
+			Id:          jsonAlert.Get("id").MustInt64(),
+			Name:        jsonAlert.Get("name").MustString(),
+			Handler:     jsonAlert.Get("handler").MustInt64(),
+			Message:     jsonAlert.Get("message").MustString(),
+			Frequency:   frequency,
+			For:         forValue,
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {
diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index ac1885fb380..d2a505145ac 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -20,7 +20,7 @@ type Rule struct {
 	Name                string
 	Message             string
 	LastStateChange     time.Time
-	DebounceDuration    time.Duration
+	For                 time.Duration
 	NoDataState         m.NoDataOption
 	ExecutionErrorState m.ExecutionErrorOption
 	State               m.AlertStateType
@@ -104,7 +104,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
 	model.LastStateChange = ruleDef.NewStateDate
-	model.DebounceDuration = ruleDef.DebounceDuration
+	model.For = ruleDef.For
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index f575fb3b02b..b5aeb26483c 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -134,7 +134,7 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
 		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
 
-	mg.AddMigration("Add decounce_duration to alert table", NewAddColumnMigration(alertV1, &Column{
-		Name: "debounce_duration", Type: DB_BigInt, Nullable: true,
+	mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
+		Name: "for", Type: DB_BigInt, Nullable: true,
 	}))
 }
diff --git a/public/app/features/alerting/AlertTabCtrl.ts b/public/app/features/alerting/AlertTabCtrl.ts
index 43752d0a8c9..758b3273d1a 100644
--- a/public/app/features/alerting/AlertTabCtrl.ts
+++ b/public/app/features/alerting/AlertTabCtrl.ts
@@ -169,7 +169,7 @@ export class AlertTabCtrl {
     alert.frequency = alert.frequency || '1m';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
-    alert.debounceDuration = alert.debounceDuration || '5m';
+    alert.for = alert.for || '5m';
 
     const defaultName = this.panel.title + ' alert';
     alert.name = alert.name || defaultName;
diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html
index de5fc4df382..676a1d32937 100644
--- a/public/app/features/alerting/partials/alert_tab.html
+++ b/public/app/features/alerting/partials/alert_tab.html
@@ -28,16 +28,16 @@
         <h5 class="section-heading">Alert Config</h5>
         <div class="gf-form">
           <span class="gf-form-label width-6">Name</span>
-          <input type="text" class="gf-form-input width-22" ng-model="ctrl.alert.name">
+          <input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
         </div>
         <div class="gf-form-inline">
           <div class="gf-form">
-            <span class="gf-form-label width-8">Evaluate every</span>
-            <input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency">
+            <span class="gf-form-label width-9">Evaluate every</span>
+            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
           </div>
-          <div class="gf-form max-width-15">
-            <label class="gf-form-label width-10">Debounce duration</label>
-            <input type="text" class="gf-form-input max-width-5" ng-model="ctrl.alert.debounceDuration" spellcheck='false' placeholder="5m">
+          <div class="gf-form max-width-11">
+            <label class="gf-form-label width-5">For</label>
+            <input type="text" class="gf-form-input max-width-6" ng-model="ctrl.alert.for" spellcheck='false' placeholder="5m">
             <info-popover mode="right-absolute">
               Configuring this value means that an alert rule have to be firing for atleast this duration before changing state.
               This should reduce false positive alerts and avoid flapping alerts.

From ae2d536740136ba538b5d24605b48905dacfdc8e Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 5 Nov 2018 13:14:02 +0100
Subject: [PATCH 07/23] adds tests for extracting for property

---
 pkg/services/alerting/extractor.go            |  9 ++++-----
 pkg/services/alerting/extractor_test.go       | 20 ++++++++++++-------
 .../collapsed-panels.json                     |  0
 .../dash-without-id.json                      |  0
 .../graphite-alert.json                       |  1 +
 .../influxdb-alert.json                       |  0
 .../panel-with-id-0.json                      |  0
 .../panels-missing-id.json                    |  0
 .../{test-data => testdata}/v5-dashboard.json |  0
 9 files changed, 18 insertions(+), 12 deletions(-)
 rename pkg/services/alerting/{test-data => testdata}/collapsed-panels.json (100%)
 rename pkg/services/alerting/{test-data => testdata}/dash-without-id.json (100%)
 rename pkg/services/alerting/{test-data => testdata}/graphite-alert.json (98%)
 rename pkg/services/alerting/{test-data => testdata}/influxdb-alert.json (100%)
 rename pkg/services/alerting/{test-data => testdata}/panel-with-id-0.json (100%)
 rename pkg/services/alerting/{test-data => testdata}/panels-missing-id.json (100%)
 rename pkg/services/alerting/{test-data => testdata}/v5-dashboard.json (100%)

diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go
index 244dc0a0770..0d902b388a8 100644
--- a/pkg/services/alerting/extractor.go
+++ b/pkg/services/alerting/extractor.go
@@ -2,9 +2,8 @@ package alerting
 
 import (
 	"errors"
-	"time"
-
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -114,10 +113,10 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			return nil, ValidationError{Reason: "Could not parse frequency"}
 		}
 
-		rawFow := jsonAlert.Get("for").MustString()
+		rawFor := jsonAlert.Get("for").MustString()
 		var forValue time.Duration
-		if rawFow != "" {
-			forValue, err = time.ParseDuration(rawFow)
+		if rawFor != "" {
+			forValue, err = time.ParseDuration(rawFor)
 			if err != nil {
 				return nil, ValidationError{Reason: "Could not parse for"}
 			}
diff --git a/pkg/services/alerting/extractor_test.go b/pkg/services/alerting/extractor_test.go
index e2dc01a1181..d03565ead90 100644
--- a/pkg/services/alerting/extractor_test.go
+++ b/pkg/services/alerting/extractor_test.go
@@ -3,6 +3,7 @@ package alerting
 import (
 	"io/ioutil"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 			return nil
 		})
 
-		json, err := ioutil.ReadFile("./test-data/graphite-alert.json")
+		json, err := ioutil.ReadFile("./testdata/graphite-alert.json")
 		So(err, ShouldBeNil)
 
 		Convey("Extractor should not modify the original json", func() {
@@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) {
 					So(alerts[1].PanelId, ShouldEqual, 4)
 				})
 
+				Convey("should extract for param", func() {
+					So(alerts[0].For, ShouldEqual, time.Minute*2)
+					So(alerts[1].For, ShouldEqual, time.Duration(0))
+				})
+
 				Convey("should extract name and desc", func() {
 					So(alerts[0].Name, ShouldEqual, "name1")
 					So(alerts[0].Message, ShouldEqual, "desc1")
@@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panels missing id should return error", func() {
-			panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json")
+			panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithoutId)
@@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panel with id set to zero should return error", func() {
-			panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
+			panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithIdZero)
@@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse alerts from dashboard without rows", func() {
-			json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
+			json, err := ioutil.ReadFile("./testdata/v5-dashboard.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard containing influxdb alert", func() {
-			json, err := ioutil.ReadFile("./test-data/influxdb-alert.json")
+			json, err := ioutil.ReadFile("./testdata/influxdb-alert.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Should be able to extract collapsed panels", func() {
-			json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
+			json, err := ioutil.ReadFile("./testdata/collapsed-panels.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard without id and containing an alert", func() {
-			json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
+			json, err := ioutil.ReadFile("./testdata/dash-without-id.json")
 			So(err, ShouldBeNil)
 
 			dashJSON, err := simplejson.NewJson(json)
diff --git a/pkg/services/alerting/test-data/collapsed-panels.json b/pkg/services/alerting/testdata/collapsed-panels.json
similarity index 100%
rename from pkg/services/alerting/test-data/collapsed-panels.json
rename to pkg/services/alerting/testdata/collapsed-panels.json
diff --git a/pkg/services/alerting/test-data/dash-without-id.json b/pkg/services/alerting/testdata/dash-without-id.json
similarity index 100%
rename from pkg/services/alerting/test-data/dash-without-id.json
rename to pkg/services/alerting/testdata/dash-without-id.json
diff --git a/pkg/services/alerting/test-data/graphite-alert.json b/pkg/services/alerting/testdata/graphite-alert.json
similarity index 98%
rename from pkg/services/alerting/test-data/graphite-alert.json
rename to pkg/services/alerting/testdata/graphite-alert.json
index 5f23e224f9a..3cb4ae1dd22 100644
--- a/pkg/services/alerting/test-data/graphite-alert.json
+++ b/pkg/services/alerting/testdata/graphite-alert.json
@@ -23,6 +23,7 @@
           "message": "desc1",
           "handler": 1,
           "frequency": "60s",
+          "for": "2m",
           "conditions": [
           {
             "type": "query",
diff --git a/pkg/services/alerting/test-data/influxdb-alert.json b/pkg/services/alerting/testdata/influxdb-alert.json
similarity index 100%
rename from pkg/services/alerting/test-data/influxdb-alert.json
rename to pkg/services/alerting/testdata/influxdb-alert.json
diff --git a/pkg/services/alerting/test-data/panel-with-id-0.json b/pkg/services/alerting/testdata/panel-with-id-0.json
similarity index 100%
rename from pkg/services/alerting/test-data/panel-with-id-0.json
rename to pkg/services/alerting/testdata/panel-with-id-0.json
diff --git a/pkg/services/alerting/test-data/panels-missing-id.json b/pkg/services/alerting/testdata/panels-missing-id.json
similarity index 100%
rename from pkg/services/alerting/test-data/panels-missing-id.json
rename to pkg/services/alerting/testdata/panels-missing-id.json
diff --git a/pkg/services/alerting/test-data/v5-dashboard.json b/pkg/services/alerting/testdata/v5-dashboard.json
similarity index 100%
rename from pkg/services/alerting/test-data/v5-dashboard.json
rename to pkg/services/alerting/testdata/v5-dashboard.json

From 3789583014fc6d1e2de0eb03b479953feea0e696 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 5 Nov 2018 13:51:35 +0100
Subject: [PATCH 08/23] for: use 0m as default for existing alerts and 5m for
 new

---
 public/app/features/alerting/AlertTabCtrl.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/public/app/features/alerting/AlertTabCtrl.ts b/public/app/features/alerting/AlertTabCtrl.ts
index 758b3273d1a..2efd3c062c4 100644
--- a/public/app/features/alerting/AlertTabCtrl.ts
+++ b/public/app/features/alerting/AlertTabCtrl.ts
@@ -169,7 +169,7 @@ export class AlertTabCtrl {
     alert.frequency = alert.frequency || '1m';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
-    alert.for = alert.for || '5m';
+    alert.for = alert.for || '0m';
 
     const defaultName = this.panel.title + ' alert';
     alert.name = alert.name || defaultName;
@@ -355,6 +355,7 @@ export class AlertTabCtrl {
   enable() {
     this.panel.alert = {};
     this.initModel();
+    this.panel.alert.for = '5m'; //default value for new alerts. for existing alerts we use 0m to avoid breaking changes
   }
 
   evaluatorParamsChanged() {

From 0ddfd92f8c5fa9c3a4a2204de21f76db21c8c73c Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Wed, 7 Nov 2018 22:30:23 +0100
Subject: [PATCH 09/23] adds debounce duration for alert dashboards in ha_test

---
 devenv/docker/ha_test/docker-compose.yaml                 | 2 +-
 devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml
index ce8630d88a4..1195e2a977c 100644
--- a/devenv/docker/ha_test/docker-compose.yaml
+++ b/devenv/docker/ha_test/docker-compose.yaml
@@ -9,7 +9,7 @@ services:
       - /var/run/docker.sock:/tmp/docker.sock:ro
 
   db:
-    image: mysql
+    image: mysql:5.6
     environment:
       MYSQL_ROOT_PASSWORD: rootpass
       MYSQL_DATABASE: grafana
diff --git a/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet
index 86ded7e79d6..e9b8abfbb9c 100644
--- a/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet
+++ b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet
@@ -39,6 +39,7 @@ local alertDashboardTemplate = {
         "executionErrorState": "alerting",
         "frequency": "10s",
         "handler": 1,
+        "for": "1m",
         "name": "bulk alerting",
         "noDataState": "no_data",
         "notifications": [

From 975f0aa064634c9181418cf50e8598d5a0cb747a Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 5 Nov 2018 11:25:37 +0100
Subject: [PATCH 10/23] alerting: adds docs about the for setting

---
 docs/sources/alerting/rules.md | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md
index 488619055e2..2e4a7e5c191 100644
--- a/docs/sources/alerting/rules.md
+++ b/docs/sources/alerting/rules.md
@@ -39,7 +39,7 @@ Currently alerting supports a limited form of high availability. Since v4.2.0 of
 
 ## Rule Config
 
-{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
+
 
 Currently only the graph panel supports alert rules but this will be added to the **Singlestat** and **Table**
 panels as well in a future release.
@@ -48,6 +48,16 @@ panels as well in a future release.
 
 Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
 
+### For
+
+> This setting is available in Grafana 5.4 and above.
+
+The `For` setting allows you to specify a duration for which the alert has to violate the threshold before switching to `Alerting` state and sending notifications. This is useful when you want to reduce the amount of false positive alerts and problems from which the system selfheal. Which in case a human does not need to be woken up.
+
+Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers.
+
+{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
+
 ### Conditions
 
 Currently the only condition type that exists is a `Query` condition that allows you to
@@ -57,11 +67,11 @@ specify a query letter, time range and an aggregation function.
 ### Query condition example
 
 ```sql
-avg() OF query(A, 5m, now) IS BELOW 14
+avg() OF query(A, 15m, now) IS BELOW 14
 ```
 
 - `avg()` Controls how the values for **each** series should be reduced to a value that can be compared against the threshold. Click on the function to change it to another aggregation function.
-- `query(A, 5m, now)`  The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `5m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
+- `query(A, 15m, now)`  The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `15m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
 - `IS BELOW 14`  Defines the type of threshold and the threshold value.  You can click on `IS BELOW` to change the type of threshold.
 
 The query used in an alert rule cannot contain any template variables. Currently we only support `AND` and `OR` operators between conditions and they are executed serially.

From aa1b80fe45b2a8fdf7afcb3e62e6ba54cbed112a Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 8 Nov 2018 14:16:58 +0100
Subject: [PATCH 11/23] docs: improve helper test for `For`

---
 docs/sources/alerting/rules.md                       | 2 +-
 public/app/features/alerting/partials/alert_tab.html | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md
index 2e4a7e5c191..387132ecfeb 100644
--- a/docs/sources/alerting/rules.md
+++ b/docs/sources/alerting/rules.md
@@ -52,7 +52,7 @@ Here you can specify the name of the alert rule and how often the scheduler shou
 
 > This setting is available in Grafana 5.4 and above.
 
-The `For` setting allows you to specify a duration for which the alert has to violate the threshold before switching to `Alerting` state and sending notifications. This is useful when you want to reduce the amount of false positive alerts and problems from which the system selfheal. Which in case a human does not need to be woken up.
+If an alert rule has a configured `For` and the query violates the configured threshold it will first go from `OK` to `Pending`. Going from `OK` to `Pending` Grafana will not send any notifications. Once the alert rule has been firing for more than `For` duration, it will change to `Alerting` and send alert notifications. 
 
 Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers.
 
diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html
index 676a1d32937..2b5e9bdf0cb 100644
--- a/public/app/features/alerting/partials/alert_tab.html
+++ b/public/app/features/alerting/partials/alert_tab.html
@@ -39,8 +39,8 @@
             <label class="gf-form-label width-5">For</label>
             <input type="text" class="gf-form-input max-width-6" ng-model="ctrl.alert.for" spellcheck='false' placeholder="5m">
             <info-popover mode="right-absolute">
-              Configuring this value means that an alert rule have to be firing for atleast this duration before changing state.
-              This should reduce false positive alerts and avoid flapping alerts.
+                If an alert rule has a configured For and the query violates the configured threshold it will first go from OK to Pending. 
+                Going from OK to Pending Grafana will not send any notifications. Once the alert rule has been firing for more than For duration, it will change to Alerting and send alert notifications. 
             </info-popover>
           </div>
         </div>

From 2fb78a50d6a1572debb2f1227adaba1f29295af3 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 12 Nov 2018 10:50:56 +0100
Subject: [PATCH 12/23] minor fixes based on code review

---
 pkg/services/alerting/eval_context.go | 4 ++--
 pkg/services/sqlstore/alert.go        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 208fe1d188b..23d3efa8bea 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -70,11 +70,11 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 		}
 	case m.AlertStateUnknown:
 		return &StateDescription{
-			Color: "888888",
+			Color: "#888888",
 			Text:  "Unknown",
 		}
 	default:
-		panic("Unknown rule state for alert notifications " + c.Rule.State)
+		panic("Unknown rule state for alert " + c.Rule.State)
 	}
 }
 
diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go
index 78a71cc8497..62ab348664f 100644
--- a/pkg/services/sqlstore/alert.go
+++ b/pkg/services/sqlstore/alert.go
@@ -193,7 +193,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message", "debounce_duration")
+				sess.MustCols("message", "for")
 
 				_, err := sess.ID(alert.Id).Update(alert)
 				if err != nil {

From 1958de72207b15eb0b55604dca0bcdc37dfcaa07 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Tue, 13 Nov 2018 11:51:06 +0100
Subject: [PATCH 13/23] devenv: update alerting with testdata dashboard

---
 devenv/dev-dashboards/testdata_alerts.json | 796 ++++++++++++++-------
 1 file changed, 543 insertions(+), 253 deletions(-)

diff --git a/devenv/dev-dashboards/testdata_alerts.json b/devenv/dev-dashboards/testdata_alerts.json
index 8c2edebf155..8fd7d4d9db5 100644
--- a/devenv/dev-dashboards/testdata_alerts.json
+++ b/devenv/dev-dashboards/testdata_alerts.json
@@ -1,250 +1,546 @@
 {
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                60
+              ],
+              "type": "gt"
+            },
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "enabled": true,
+        "frequency": "60s",
+        "handler": 1,
+        "name": "TestData - Always OK",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "1,20,90,30,5,0",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 60
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always OK",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": "125",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                177
+              ],
+              "type": "gt"
+            },
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "enabled": true,
+        "executionErrorState": "alerting",
+        "for": "0m",
+        "frequency": "60s",
+        "handler": 1,
+        "name": "TestData - Always Alerting",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "200,445,100,150,200,220,190",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 177
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always Alerting",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                1
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "15m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "for": "5m",
+        "frequency": "1m",
+        "handler": 1,
+        "name": "TestData - No data",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "id": 5,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "no_data_points",
+          "stringInput": "",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 1
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "No data",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                177
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "15m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "for": "1m",
+        "frequency": "1m",
+        "handler": 1,
+        "name": "TestData - Always Alerting with For",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "id": 6,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "connected",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "refId": "A",
+          "scenario": "random_walk",
+          "scenarioId": "csv_metric_values",
+          "stringInput": "200,445,100,150,200,220,190",
+          "target": ""
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 177
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Always Alerting with For",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
   "revision": 2,
-  "title": "Alerting with TestData",
+  "schemaVersion": 16,
+  "style": "dark",
   "tags": [
     "grafana-test"
   ],
-  "style": "dark",
-  "timezone": "browser",
-  "editable": true,
-  "hideControls": false,
-  "sharedCrosshair": false,
-  "rows": [
-    {
-      "collapse": false,
-      "editable": true,
-      "height": 255.625,
-      "panels": [
-        {
-          "alert": {
-            "conditions": [
-              {
-                "evaluator": {
-                  "params": [
-                    60
-                  ],
-                  "type": "gt"
-                },
-                "query": {
-                  "params": [
-                    "A",
-                    "5m",
-                    "now"
-                  ]
-                },
-                "reducer": {
-                  "params": [],
-                  "type": "avg"
-                },
-                "type": "query"
-              }
-            ],
-            "enabled": true,
-            "frequency": "60s",
-            "handler": 1,
-            "name": "TestData - Always OK",
-            "noDataState": "no_data",
-            "notifications": []
-          },
-          "aliasColors": {},
-          "bars": false,
-          "datasource": "gdev-testdata",
-          "editable": true,
-          "error": false,
-          "fill": 1,
-          "id": 3,
-          "isNew": true,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 2,
-          "links": [],
-          "nullPointMode": "connected",
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "span": 6,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "refId": "A",
-              "scenario": "random_walk",
-              "scenarioId": "csv_metric_values",
-              "stringInput": "1,20,90,30,5,0",
-              "target": ""
-            }
-          ],
-          "thresholds": [
-            {
-              "value": 60,
-              "op": "gt",
-              "fill": true,
-              "line": true,
-              "colorMode": "critical"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Always OK",
-          "tooltip": {
-            "msResolution": false,
-            "shared": true,
-            "sort": 0,
-            "value_type": "cumulative"
-          },
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": "125",
-              "min": "0",
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ]
-        },
-        {
-          "alert": {
-            "conditions": [
-              {
-                "evaluator": {
-                  "params": [
-                    177
-                  ],
-                  "type": "gt"
-                },
-                "query": {
-                  "params": [
-                    "A",
-                    "5m",
-                    "now"
-                  ]
-                },
-                "reducer": {
-                  "params": [],
-                  "type": "avg"
-                },
-                "type": "query"
-              }
-            ],
-            "enabled": true,
-            "frequency": "60s",
-            "handler": 1,
-            "name": "TestData - Always Alerting",
-            "noDataState": "no_data",
-            "notifications": []
-          },
-          "aliasColors": {},
-          "bars": false,
-          "datasource": "gdev-testdata",
-          "editable": true,
-          "error": false,
-          "fill": 1,
-          "id": 4,
-          "isNew": true,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 2,
-          "links": [],
-          "nullPointMode": "connected",
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "span": 6,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "refId": "A",
-              "scenario": "random_walk",
-              "scenarioId": "csv_metric_values",
-              "stringInput": "200,445,100,150,200,220,190",
-              "target": ""
-            }
-          ],
-          "thresholds": [
-            {
-              "colorMode": "critical",
-              "fill": true,
-              "line": true,
-              "op": "gt",
-              "value": 177
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Always Alerting",
-          "tooltip": {
-            "msResolution": false,
-            "shared": true,
-            "sort": 0,
-            "value_type": "cumulative"
-          },
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": "",
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ]
-        }
-      ],
-      "title": "New row"
-    }
-  ],
+  "templating": {
+    "list": []
+  },
   "time": {
     "from": "now-6h",
     "to": "now"
@@ -274,14 +570,8 @@
       "30d"
     ]
   },
-  "templating": {
-    "list": []
-  },
-  "annotations": {
-    "list": []
-  },
-  "schemaVersion": 13,
-  "version": 4,
-  "links": [],
-  "gnetId": null
-}
+  "timezone": "browser",
+  "title": "Alerting with TestData",
+  "uid": "7MeksYbmk",
+  "version": 1
+}
\ No newline at end of file

From 8fb997d935e47798879d1e6b03daefe51b2370d8 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Wed, 14 Nov 2018 23:19:35 +0100
Subject: [PATCH 14/23] should not notify when going from unknown to pending

---
 pkg/services/alerting/notifiers/base.go      |  5 +++++
 pkg/services/alerting/notifiers/base_test.go | 20 +++++++++++---------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 35d3ff518a0..d4a9975bcba 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -71,6 +71,11 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
 		return false
 	}
 
+	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
+		return false
+	}
+
 	// Do not notify when we become OK from pending
 	if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
 		return false
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 388c2db17ee..3fd4447eefe 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStatePending,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateAlerting,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStatePending,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateAlerting,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState:    m.AlertStateAlerting,
 			frequency:    time.Minute * 10,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -138,7 +131,13 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			name:      "unknown -> ok",
 			prevState: m.AlertStateUnknown,
 			newState:  m.AlertStateOK,
-			state:     &m.AlertNotificationState{},
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> pending",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
 
 			expect: false,
 		},
@@ -146,7 +145,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			name:      "unknown -> alerting",
 			prevState: m.AlertStateUnknown,
 			newState:  m.AlertStateAlerting,
-			state:     &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -157,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			State: tc.prevState,
 		})
 
+		if tc.state == nil {
+			tc.state = &m.AlertNotificationState{}
+		}
+
 		evalContext.Rule.State = tc.newState
 		nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}
 

From 84eb3bd0958ca4606876da45c39fe1da70385ae4 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Wed, 14 Nov 2018 23:39:44 +0100
Subject: [PATCH 15/23] tests for supporting for with all alerting scenarios

---
 pkg/services/alerting/eval_context_test.go | 44 ++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go
index cc0bed79d10..d9615ee4801 100644
--- a/pkg/services/alerting/eval_context_test.go
+++ b/pkg/services/alerting/eval_context_test.go
@@ -139,6 +139,50 @@ func TestGetStateFromEvalContext(t *testing.T) {
 				ec.NoDataFound = true
 			},
 		},
+		{
+			name:     "pending -> no_data(alerting) with for duration have not passed",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> no_data(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) with for duration have not passed ",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
 	}
 
 	for _, tc := range tcs {

From 28029ce4a7e6850e8a5665343507d200e9102831 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 11:04:16 +0100
Subject: [PATCH 16/23] alerting: support `for` on execution errors and notdata

---
 pkg/services/alerting/eval_context.go | 32 ++++++++++++++++-----------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 23d3efa8bea..5a4b378ac28 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -118,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
 	return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil
 }
 
+// GetNewState returns the new state from the alert rule evaluation
 func (c *EvalContext) GetNewState() m.AlertStateType {
+	ns := getNewStateInternal(c)
+	if ns != m.AlertStateAlerting || c.Rule.For == 0 {
+		return ns
+	}
+
+	since := time.Now().Sub(c.Rule.LastStateChange)
+	if since > c.Rule.For {
+		return m.AlertStateAlerting
+	}
+
+	if c.PrevAlertState == m.AlertStateAlerting {
+		return m.AlertStateAlerting
+	}
+
+	return m.AlertStatePending
+}
+
+func getNewStateInternal(c *EvalContext) m.AlertStateType {
 	if c.Error != nil {
 		c.log.Error("Alert Rule Result Error",
 			"ruleId", c.Rule.Id,
@@ -132,19 +151,6 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 		return c.Rule.ExecutionErrorState.ToAlertState()
 	}
 
-	if c.Firing && c.Rule.For != 0 {
-		since := time.Now().Sub(c.Rule.LastStateChange)
-		if since > c.Rule.For {
-			return m.AlertStateAlerting
-		}
-
-		if c.PrevAlertState == m.AlertStateAlerting {
-			return m.AlertStateAlerting
-		}
-
-		return m.AlertStatePending
-	}
-
 	if c.Firing {
 		return m.AlertStateAlerting
 	}

From a70ea2101c3ddf161786dd0c9a232d73755c1daa Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 12:36:11 +0100
Subject: [PATCH 17/23] alertmanager: adds tests for should notify

---
 .../alerting/notifiers/alertmanager_test.go   | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go
index 3549b536e48..7510742ed17 100644
--- a/pkg/services/alerting/notifiers/alertmanager_test.go
+++ b/pkg/services/alerting/notifiers/alertmanager_test.go
@@ -1,13 +1,60 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestWhenAlertManagerShouldNotify(t *testing.T) {
+	tcs := []struct {
+		prevState m.AlertStateType
+		newState  m.AlertStateType
+
+		expect bool
+	}{
+		{
+			prevState: m.AlertStatePending,
+			newState:  m.AlertStateOK,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateAlerting,
+			newState:  m.AlertStateOK,
+			expect:    true,
+		},
+		{
+			prevState: m.AlertStateOK,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+	}
+
+	for _, tc := range tcs {
+		am := &AlertmanagerNotifier{log: log.New("test.logger")}
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+			State: tc.prevState,
+		})
+
+		evalContext.Rule.State = tc.newState
+
+		res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
+		if res != tc.expect {
+			t.Errorf("got %v expected %v", res, tc.expect)
+		}
+	}
+}
+
 func TestAlertmanagerNotifier(t *testing.T) {
 	Convey("Alertmanager notifier tests", t, func() {
 

From 968bfd01391b760b3a5437a7a507447f22bbc4c7 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 12:42:47 +0100
Subject: [PATCH 18/23] adds pending state to alert list panel

---
 public/app/plugins/panel/alertlist/editor.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/public/app/plugins/panel/alertlist/editor.html b/public/app/plugins/panel/alertlist/editor.html
index c48b70e02c0..a05234cea3c 100644
--- a/public/app/plugins/panel/alertlist/editor.html
+++ b/public/app/plugins/panel/alertlist/editor.html
@@ -50,5 +50,6 @@
     <gf-form-switch class="gf-form" label="No data" label-class="width-10" checked="ctrl.stateFilter['no_data']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
     <gf-form-switch class="gf-form" label="Execution error" label-class="width-10" checked="ctrl.stateFilter['execution_error']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
     <gf-form-switch class="gf-form" label="Alerting" label-class="width-10" checked="ctrl.stateFilter['alerting']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
+    <gf-form-switch class="gf-form" label="Pending" label-class="width-10" checked="ctrl.stateFilter['pending']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
   </div>
 </div>

From e7260d77b3938188521cd2564f5f4a6a5e512371 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 12:46:28 +0100
Subject: [PATCH 19/23] adds pending filter for alert list page

---
 public/app/features/alerting/AlertRuleList.tsx       |  1 +
 .../__snapshots__/AlertRuleList.test.tsx.snap        | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/public/app/features/alerting/AlertRuleList.tsx b/public/app/features/alerting/AlertRuleList.tsx
index d25fc659af5..f94134f3ee1 100644
--- a/public/app/features/alerting/AlertRuleList.tsx
+++ b/public/app/features/alerting/AlertRuleList.tsx
@@ -29,6 +29,7 @@ export class AlertRuleList extends PureComponent<Props, any> {
     { text: 'Alerting', value: 'alerting' },
     { text: 'No Data', value: 'no_data' },
     { text: 'Paused', value: 'paused' },
+    { text: 'Pending', value: 'pending' },
   ];
 
   componentDidMount() {
diff --git a/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap b/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap
index 4ae27213e1e..b753a852e92 100644
--- a/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap
+++ b/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap
@@ -81,6 +81,12 @@ exports[`Render should render alert rules 1`] = `
             >
               Paused
             </option>
+            <option
+              key="pending"
+              value="pending"
+            >
+              Pending
+            </option>
           </select>
         </div>
       </div>
@@ -230,6 +236,12 @@ exports[`Render should render component 1`] = `
             >
               Paused
             </option>
+            <option
+              key="pending"
+              value="pending"
+            >
+              Pending
+            </option>
           </select>
         </div>
       </div>

From 7ba04466a2b305b01d1a7832bed24881b4ce6e59 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 14:30:36 +0100
Subject: [PATCH 20/23] alerting: improve annotations for pending state

---
 public/app/core/utils/colors.ts                  | 1 +
 public/app/features/annotations/event_manager.ts | 6 ++++++
 public/app/features/panel/panel_directive.ts     | 6 +++++-
 public/sass/pages/_alerting.scss                 | 7 +++++++
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/public/app/core/utils/colors.ts b/public/app/core/utils/colors.ts
index e8a7366beb5..2ab79ba27fb 100644
--- a/public/app/core/utils/colors.ts
+++ b/public/app/core/utils/colors.ts
@@ -7,6 +7,7 @@ export const DEFAULT_ANNOTATION_COLOR = 'rgba(0, 211, 255, 1)';
 export const OK_COLOR = 'rgba(11, 237, 50, 1)';
 export const ALERTING_COLOR = 'rgba(237, 46, 24, 1)';
 export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)';
+export const PENDING_COLOR = 'rgba(247, 149, 32, 1)';
 export const REGION_FILL_ALPHA = 0.09;
 
 const colors = [
diff --git a/public/app/features/annotations/event_manager.ts b/public/app/features/annotations/event_manager.ts
index ef74ca193d4..db748e639a1 100644
--- a/public/app/features/annotations/event_manager.ts
+++ b/public/app/features/annotations/event_manager.ts
@@ -7,6 +7,7 @@ import {
   OK_COLOR,
   ALERTING_COLOR,
   NO_DATA_COLOR,
+  PENDING_COLOR,
   DEFAULT_ANNOTATION_COLOR,
   REGION_FILL_ALPHA,
 } from 'app/core/utils/colors';
@@ -71,6 +72,11 @@ export class EventManager {
         position: 'BOTTOM',
         markerSize: 5,
       },
+      $__pending: {
+        color: PENDING_COLOR,
+        position: 'BOTTOM',
+        markerSize: 5,
+      },
       $__editing: {
         color: DEFAULT_ANNOTATION_COLOR,
         position: 'BOTTOM',
diff --git a/public/app/features/panel/panel_directive.ts b/public/app/features/panel/panel_directive.ts
index 77ebf754b3a..aef7ca5e256 100644
--- a/public/app/features/panel/panel_directive.ts
+++ b/public/app/features/panel/panel_directive.ts
@@ -161,7 +161,11 @@ module.directive('grafanaPanel', ($rootScope, $document, $timeout) => {
             panelContainer.removeClass('panel-alert-state--' + lastAlertState);
           }
 
-          if (ctrl.alertState.state === 'ok' || ctrl.alertState.state === 'alerting') {
+          if (
+            ctrl.alertState.state === 'ok' ||
+            ctrl.alertState.state === 'alerting' ||
+            ctrl.alertState.state === 'pending'
+          ) {
             panelContainer.addClass('panel-alert-state--' + ctrl.alertState.state);
           }
 
diff --git a/public/sass/pages/_alerting.scss b/public/sass/pages/_alerting.scss
index 90f2eb526f1..77752be11bc 100644
--- a/public/sass/pages/_alerting.scss
+++ b/public/sass/pages/_alerting.scss
@@ -66,6 +66,13 @@
       content: '\e611';
     }
   }
+
+  &--pending {
+    .panel-alert-icon:before {
+      color: $warn;
+      content: '\e611';
+    }
+  }
 }
 
 @keyframes alerting-panel {

From caec36e7ece559cf213eb9ed240389ca56ed4c53 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 15 Nov 2018 15:37:46 +0100
Subject: [PATCH 21/23] alert rule have to be pending before alerting is for is
 specified

---
 pkg/services/alerting/eval_context.go      |  2 +-
 pkg/services/alerting/eval_context_test.go | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 5a4b378ac28..17ed448bd2a 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -126,7 +126,7 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 	}
 
 	since := time.Now().Sub(c.Rule.LastStateChange)
-	if since > c.Rule.For {
+	if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For {
 		return m.AlertStateAlerting
 	}
 
diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go
index d9615ee4801..4c9b88f1881 100644
--- a/pkg/services/alerting/eval_context_test.go
+++ b/pkg/services/alerting/eval_context_test.go
@@ -66,8 +66,8 @@ func TestGetStateFromEvalContext(t *testing.T) {
 			},
 		},
 		{
-			name:     "ok -> alerting. since its been firing for more than FOR",
-			expected: models.AlertStateAlerting,
+			name:     "ok -> pending. since it has to be pending longer than FOR and prev state is ok",
+			expected: models.AlertStatePending,
 			applyFn: func(ec *EvalContext) {
 				ec.PrevAlertState = models.AlertStateOK
 				ec.Firing = true
@@ -75,6 +75,16 @@ func TestGetStateFromEvalContext(t *testing.T) {
 				ec.Rule.For = time.Minute * 2
 			},
 		},
+		{
+			name:     "pending -> alerting. since its been firing for more than FOR and prev state is pending",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.For = time.Minute * 2
+			},
+		},
 		{
 			name:     "alerting -> alerting. should not update regardless of FOR",
 			expected: models.AlertStateAlerting,

From 48905a613dc37cc3ea8e15cad51319e166d203b4 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Thu, 15 Nov 2018 16:00:32 +0100
Subject: [PATCH 22/23] fix pending alert annotation tooltip icon

---
 public/app/features/annotations/annotation_tooltip.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/app/features/annotations/annotation_tooltip.ts b/public/app/features/annotations/annotation_tooltip.ts
index 16c18005204..fbe85856f31 100644
--- a/public/app/features/annotations/annotation_tooltip.ts
+++ b/public/app/features/annotations/annotation_tooltip.ts
@@ -32,7 +32,7 @@ export function annotationTooltipDirective($sanitize, dashboardSrv, contextSrv,
       if (event.alertId) {
         const stateModel = alertDef.getStateDisplayModel(event.newState);
         titleStateClass = stateModel.stateClass;
-        title = `<i class="icon-gf ${stateModel.iconClass}"></i> ${stateModel.text}`;
+        title = `<i class="${stateModel.iconClass}"></i> ${stateModel.text}`;
         text = alertDef.getAlertAnnotationInfo(event);
         if (event.text) {
           text = text + '<br />' + event.text;

From 76cbd7f0de481743c327ef3832527c3e1f38f1bf Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 19 Nov 2018 09:10:19 +0100
Subject: [PATCH 23/23] alerting: reduce the length of range queries

since we introduce deboucing the length should matter less.
But going below 5m as default might be weird depending on datasource.
---
 public/app/features/alerting/AlertTabCtrl.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/app/features/alerting/AlertTabCtrl.ts b/public/app/features/alerting/AlertTabCtrl.ts
index 2efd3c062c4..ef68ddcf4a5 100644
--- a/public/app/features/alerting/AlertTabCtrl.ts
+++ b/public/app/features/alerting/AlertTabCtrl.ts
@@ -218,7 +218,7 @@ export class AlertTabCtrl {
   buildDefaultCondition() {
     return {
       type: 'query',
-      query: { params: ['A', '15m', 'now'] },
+      query: { params: ['A', '5m', 'now'] },
       reducer: { type: 'avg', params: [] },
       evaluator: { type: 'gt', params: [null] },
       operator: { type: 'and' },