Merge pull request #13947 from bergquist/alerting_for

Introduce alert debouncing

pkg/services/alerting/notifiers/alertmanager_test.go

@@ -1,13 +1,60 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestWhenAlertManagerShouldNotify(t *testing.T) {
+	tcs := []struct {
+		prevState m.AlertStateType
+		newState  m.AlertStateType
+
+		expect bool
+	}{
+		{
+			prevState: m.AlertStatePending,
+			newState:  m.AlertStateOK,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateAlerting,
+			newState:  m.AlertStateOK,
+			expect:    true,
+		},
+		{
+			prevState: m.AlertStateOK,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+	}
+
+	for _, tc := range tcs {
+		am := &AlertmanagerNotifier{log: log.New("test.logger")}
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+			State: tc.prevState,
+		})
+
+		evalContext.Rule.State = tc.newState
+
+		res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
+		if res != tc.expect {
+			t.Errorf("got %v expected %v", res, tc.expect)
+		}
+	}
+}
+
 func TestAlertmanagerNotifier(t *testing.T) {
 	Convey("Alertmanager notifier tests", t, func() {
 

pkg/services/alerting/notifiers/base.go

@@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
 	}
 
 	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
+		return false
+	}
+
+	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
+		return false
+	}
+
+	// Do not notify when we become OK from pending
 	if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
 		return false
 	}

pkg/services/alerting/notifiers/base_test.go

@@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStatePending,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateAlerting,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStatePending,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateAlerting,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState:    m.AlertStateAlerting,
 			frequency:    time.Minute * 10,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState: m.AlertStateOK,
 			state:     &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
 
+			expect: true,
+		},
+		{
+			name:      "unknown -> ok",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateOK,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> pending",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> alerting",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateAlerting,
+
 			expect: true,
 		},
 	}
@@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			State: tc.prevState,
 		})
 
+		if tc.state == nil {
+			tc.state = &m.AlertNotificationState{}
+		}
+
 		evalContext.Rule.State = tc.newState
 		nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}