public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support passing tags to Pagerduty and allow notification on specific event categories (#21335)

Browse Source 
Add support for passing tags to Pagerduty and allow notification 
on specific event categories such as Class, Group and Component.

Ref #19912, #19913

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
John Dyer
2020-01-23 13:20:07 +01:00
committed by Marcus Efraimsson
co-authored by Arve Knudsen
parent cdfac32dfd
commit 0fcb2d0b2f
28 changed files with 4023 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/notifiers/pagerduty.go
+39 -11
View File
@@ -3,6 +3,7 @@ package notifiers
import (
"os"
"strconv"
"strings"
"time"
"github.com/grafana/grafana/pkg/bus"
@@ -79,13 +80,8 @@ type PagerdutyNotifier struct {
log log.Logger
}
// Notify sends an alert notification to PagerDuty
func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
if evalContext.Rule.State == models.AlertStateOK && !pn.AutoResolve {
pn.log.Info("Not sending a trigger to Pagerduty", "state", evalContext.Rule.State, "auto resolve", pn.AutoResolve)
return nil
}
// buildEventPayload is responsible for building the event payload body for sending to Pagerduty v2 API
func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext) ([]byte, error) {
eventType := "trigger"
if evalContext.Rule.State == models.AlertStateOK {
@@ -100,6 +96,23 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
payloadJSON := simplejson.New()
// set default, override in following case switch if defined
payloadJSON.Set("component", "Grafana")
for _, tag := range evalContext.Rule.AlertRuleTags {
customData.Set(tag.Key, tag.Value)
// Override tags appropriately if they are in the PagerDuty v2 API
switch strings.ToLower(tag.Key) {
case "group":
payloadJSON.Set("group", tag.Value)
case "class":
payloadJSON.Set("class", tag.Value)
case "component":
payloadJSON.Set("component", tag.Value)
}
}
summary := evalContext.Rule.Name + " - " + evalContext.Rule.Message
if len(summary) > 1024 {
summary = summary[0:1024]
@@ -111,9 +124,7 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
}
payloadJSON.Set("severity", pn.Severity)
payloadJSON.Set("timestamp", time.Now())
payloadJSON.Set("component", "Grafana")
payloadJSON.Set("custom_details", customData)
bodyJSON := simplejson.New()
bodyJSON.Set("routing_key", pn.Key)
bodyJSON.Set("event_action", eventType)
@@ -123,13 +134,14 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
ruleURL, err := evalContext.GetRuleURL()
if err != nil {
pn.log.Error("Failed get rule link", "error", err)
return err
return []byte{}, err
}
links := make([]interface{}, 1)
linkJSON := simplejson.New()
linkJSON.Set("href", ruleURL)
bodyJSON.Set("client_url", ruleURL)
bodyJSON.Set("client", "Grafana")
links[0] = linkJSON
bodyJSON.Set("links", links)
@@ -143,6 +155,23 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
body, _ := bodyJSON.MarshalJSON()
return body, nil
}
// Notify sends an alert notification to PagerDuty
func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
if evalContext.Rule.State == models.AlertStateOK && !pn.AutoResolve {
pn.log.Info("Not sending a trigger to Pagerduty", "state", evalContext.Rule.State, "auto resolve", pn.AutoResolve)
return nil
}
body, err := pn.buildEventPayload(evalContext)
if err != nil {
pn.log.Error("Unable to build PagerDuty event payload", "error", err)
return err
}
cmd := &models.SendWebhookSync{
Url: pagerdutyEventAPIURL,
Body: string(body),
@@ -156,6 +185,5 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
pn.log.Error("Failed to send notification to Pagerduty", "error", err, "body", string(body))
return err
}
return nil
}
pkg/services/alerting/notifiers/pagerduty_test.go
+153 -10
View File
@@ -1,21 +1,34 @@
package notifiers
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
. "github.com/smartystreets/goconvey/convey"
)
func presenceComparer(a, b string) bool {
if a == "<<PRESENCE>>" {
return b != ""
}
if b == "<<PRESENCE>>" {
return a != ""
}
return a == b
}
func TestPagerdutyNotifier(t *testing.T) {
Convey("Pagerduty notifier tests", t, func() {
Convey("Parsing alert notification from settings", func() {
Convey("empty settings should return error", func() {
json := `{ }`
settingsJSON, jerr := simplejson.NewJson([]byte(json))
So(jerr, ShouldBeNil)
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pageduty_testing",
@@ -23,15 +36,15 @@ func TestPagerdutyNotifier(t *testing.T) {
Settings: settingsJSON,
}
_, err := NewPagerdutyNotifier(model)
_, err = NewPagerdutyNotifier(model)
So(err, ShouldNotBeNil)
})
Convey("severity should override default", func() {
json := `{ "integrationKey": "abcdefgh0123456789", "severity": "info" }`
json := `{ "integrationKey": "abcdefgh0123456789", "severity": "info", "tags": ["foo"]}`
settingsJSON, jerr := simplejson.NewJson([]byte(json))
So(jerr, ShouldBeNil)
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
@@ -53,8 +66,8 @@ func TestPagerdutyNotifier(t *testing.T) {
Convey("auto resolve and severity should have expected defaults", func() {
json := `{ "integrationKey": "abcdefgh0123456789" }`
settingsJSON, jerr := simplejson.NewJson([]byte(json))
So(jerr, ShouldBeNil)
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
@@ -80,8 +93,8 @@ func TestPagerdutyNotifier(t *testing.T) {
"autoResolve": false
}`
settingsJSON, jerr := simplejson.NewJson([]byte(json))
So(jerr, ShouldBeNil)
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
@@ -98,6 +111,136 @@ func TestPagerdutyNotifier(t *testing.T) {
So(pagerdutyNotifier.Key, ShouldEqual, "abcdefgh0123456789")
So(pagerdutyNotifier.AutoResolve, ShouldBeFalse)
})
Convey("should return properly formatted default v2 event payload", func() {
json := `{
"integrationKey": "abcdefgh0123456789",
"autoResolve": false
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
Type: "pagerduty",
Settings: settingsJSON,
}
not, err := NewPagerdutyNotifier(model)
So(err, ShouldBeNil)
pagerdutyNotifier := not.(*PagerdutyNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateAlerting,
})
evalContext.IsTestRun = true
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
payload, err := simplejson.NewJson(payloadJSON)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"client": "Grafana",
"client_url": "",
"dedup_key": "alertId-0",
"event_action": "trigger",
"links": []interface{}{
map[string]interface{}{
"href": "",
},
},
"payload": map[string]interface{}{
"component": "Grafana",
"source": "<<PRESENCE>>",
"custom_details": map[string]interface{}{},
"severity": "critical",
"summary": "someRule - someMessage",
"timestamp": "<<PRESENCE>>",
},
"routing_key": "abcdefgh0123456789",
}, payload.Interface(), cmp.Comparer(presenceComparer))
So(diff, ShouldBeEmpty)
})
Convey("should return properly formatted v2 event payload when using override tags", func() {
json := `{
"integrationKey": "abcdefgh0123456789",
"autoResolve": false
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
Type: "pagerduty",
Settings: settingsJSON,
}
not, err := NewPagerdutyNotifier(model)
So(err, ShouldBeNil)
pagerdutyNotifier := not.(*PagerdutyNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateAlerting,
AlertRuleTags: []*models.Tag{
{Key: "keyOnly"},
{Key: "group", Value: "aGroup"},
{Key: "class", Value: "aClass"},
{Key: "component", Value: "aComponent"},
},
})
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
evalContext.IsTestRun = true
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
payload, err := simplejson.NewJson(payloadJSON)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"client": "Grafana",
"client_url": "",
"dedup_key": "alertId-0",
"event_action": "trigger",
"links": []interface{}{
map[string]interface{}{
"href": "",
},
},
"payload": map[string]interface{}{
"source": "<<PRESENCE>>",
"component": "aComponent",
"custom_details": map[string]interface{}{
"group": "aGroup",
"class": "aClass",
"component": "aComponent",
"keyOnly": "",
},
"severity": "critical",
"summary": "someRule - someMessage",
"timestamp": "<<PRESENCE>>",
"class": "aClass",
"group": "aGroup",
},
"images": []interface{}{
map[string]interface{}{
"src": "http://somewhere.com/omg_dont_panic.png",
},
},
"routing_key": "abcdefgh0123456789",
}, payload.Interface(), cmp.Comparer(presenceComparer))
So(diff, ShouldBeEmpty)
})
})
})
}