From 8253b9ddb38e759fca520a7c1b833c3021ecc668 Mon Sep 17 00:00:00 2001 From: Andrei Lavrov Date: Mon, 7 Sep 2020 19:42:06 +0300 Subject: [PATCH] Alerting: override PagerDuty 'dedup_key' via tags (#27356) * Alerting: override PagerDuty 'dedup_key' via tags Added ability to override PagerDuty 'dedup_key' via Alert tags Updated unit tests * Updated documentation * add comma Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * add space Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/alerting/notifications.md | 3 ++- pkg/services/alerting/notifiers/pagerduty.go | 11 ++++++++--- pkg/services/alerting/notifiers/pagerduty_test.go | 5 ++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 23d2f4a730b..22783dcd2a5 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -125,13 +125,14 @@ Severity | Level for dynamic notifications, default is `critical` (1) Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2) ->**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. +>**Note:** The tags `Severity`, `Class`, `Group`, `dedup_key`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. >Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event. This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`. Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`. This behavior will become the default in a future version of Grafana. +> Using `dedup_key` tag will override grafana generated `dedup_key` with a custom key. ### Webhook The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint. diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index b3bfd07cecd..7712bcf544a 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -171,10 +171,9 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext // set default, override in following case switch if defined payloadJSON.Set("component", "Grafana") payloadJSON.Set("severity", pn.Severity) + dedupKey := "alertId-" + strconv.FormatInt(evalContext.Rule.ID, 10) for _, tag := range evalContext.Rule.AlertRuleTags { - customData.Set(tag.Key, tag.Value) - // Override tags appropriately if they are in the PagerDuty v2 API switch strings.ToLower(tag.Key) { case "group": @@ -183,6 +182,11 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext payloadJSON.Set("class", tag.Value) case "component": payloadJSON.Set("component", tag.Value) + case "dedup_key": + if len(tag.Value) > 254 { + tag.Value = tag.Value[0:254] + } + dedupKey = tag.Value case "severity": // Only set severity if it's one of the PD supported enum values // Info, Warning, Error, or Critical (case insensitive) @@ -199,6 +203,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext pn.log.Warn("Ignoring invalid severity tag", "severity", sev) } } + customData.Set(tag.Key, tag.Value) } var summary string @@ -220,7 +225,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext bodyJSON := simplejson.New() bodyJSON.Set("routing_key", pn.Key) bodyJSON.Set("event_action", eventType) - bodyJSON.Set("dedup_key", "alertId-"+strconv.FormatInt(evalContext.Rule.ID, 10)) + bodyJSON.Set("dedup_key", dedupKey) bodyJSON.Set("payload", payloadJSON) ruleURL, err := evalContext.GetRuleURL() diff --git a/pkg/services/alerting/notifiers/pagerduty_test.go b/pkg/services/alerting/notifiers/pagerduty_test.go index a5d94287beb..a182eaf1a67 100644 --- a/pkg/services/alerting/notifiers/pagerduty_test.go +++ b/pkg/services/alerting/notifiers/pagerduty_test.go @@ -2,6 +2,7 @@ package notifiers import ( "context" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -269,6 +270,7 @@ func TestPagerdutyNotifier(t *testing.T) { {Key: "class", Value: "aClass"}, {Key: "component", Value: "aComponent"}, {Key: "severity", Value: "warning"}, + {Key: "dedup_key", Value: "key-" + strings.Repeat("x", 260)}, }, }) evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png" @@ -282,7 +284,7 @@ func TestPagerdutyNotifier(t *testing.T) { diff := cmp.Diff(map[string]interface{}{ "client": "Grafana", "client_url": "", - "dedup_key": "alertId-0", + "dedup_key": "key-" + strings.Repeat("x", 250), "event_action": "trigger", "links": []interface{}{ map[string]interface{}{ @@ -297,6 +299,7 @@ func TestPagerdutyNotifier(t *testing.T) { "class": "aClass", "component": "aComponent", "severity": "warning", + "dedup_key": "key-" + strings.Repeat("x", 250), "keyOnly": "", }, "severity": "warning",