Alerting: override PagerDuty 'dedup_key' via tags (#27356)
* Alerting: override PagerDuty 'dedup_key' via tags Added ability to override PagerDuty 'dedup_key' via Alert tags Updated unit tests * Updated documentation * add comma Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * add space Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
This commit is contained in:
Andrei Lavrov
co-authored by
Diana Payton
Diana Payton
parent
e71eb13386
commit
8253b9ddb3
@@ -125,13 +125,14 @@ Severity | Level for dynamic notifications, default is `critical` (1)
|
||||
Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
|
||||
Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2)
|
||||
|
||||
>**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
|
||||
>**Note:** The tags `Severity`, `Class`, `Group`, `dedup_key`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
|
||||
|
||||
>Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event.
|
||||
This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`.
|
||||
Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`.
|
||||
This behavior will become the default in a future version of Grafana.
|
||||
|
||||
> Using `dedup_key` tag will override grafana generated `dedup_key` with a custom key.
|
||||
### Webhook
|
||||
|
||||
The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint.
|
||||
|
||||
@@ -171,10 +171,9 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
// set default, override in following case switch if defined
|
||||
payloadJSON.Set("component", "Grafana")
|
||||
payloadJSON.Set("severity", pn.Severity)
|
||||
dedupKey := "alertId-" + strconv.FormatInt(evalContext.Rule.ID, 10)
|
||||
|
||||
for _, tag := range evalContext.Rule.AlertRuleTags {
|
||||
customData.Set(tag.Key, tag.Value)
|
||||
|
||||
// Override tags appropriately if they are in the PagerDuty v2 API
|
||||
switch strings.ToLower(tag.Key) {
|
||||
case "group":
|
||||
@@ -183,6 +182,11 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
payloadJSON.Set("class", tag.Value)
|
||||
case "component":
|
||||
payloadJSON.Set("component", tag.Value)
|
||||
case "dedup_key":
|
||||
if len(tag.Value) > 254 {
|
||||
tag.Value = tag.Value[0:254]
|
||||
}
|
||||
dedupKey = tag.Value
|
||||
case "severity":
|
||||
// Only set severity if it's one of the PD supported enum values
|
||||
// Info, Warning, Error, or Critical (case insensitive)
|
||||
@@ -199,6 +203,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
|
||||
}
|
||||
}
|
||||
customData.Set(tag.Key, tag.Value)
|
||||
}
|
||||
|
||||
var summary string
|
||||
@@ -220,7 +225,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
bodyJSON := simplejson.New()
|
||||
bodyJSON.Set("routing_key", pn.Key)
|
||||
bodyJSON.Set("event_action", eventType)
|
||||
bodyJSON.Set("dedup_key", "alertId-"+strconv.FormatInt(evalContext.Rule.ID, 10))
|
||||
bodyJSON.Set("dedup_key", dedupKey)
|
||||
bodyJSON.Set("payload", payloadJSON)
|
||||
|
||||
ruleURL, err := evalContext.GetRuleURL()
|
||||
|
||||
@@ -2,6 +2,7 @@ package notifiers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -269,6 +270,7 @@ func TestPagerdutyNotifier(t *testing.T) {
|
||||
{Key: "class", Value: "aClass"},
|
||||
{Key: "component", Value: "aComponent"},
|
||||
{Key: "severity", Value: "warning"},
|
||||
{Key: "dedup_key", Value: "key-" + strings.Repeat("x", 260)},
|
||||
},
|
||||
})
|
||||
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
|
||||
@@ -282,7 +284,7 @@ func TestPagerdutyNotifier(t *testing.T) {
|
||||
diff := cmp.Diff(map[string]interface{}{
|
||||
"client": "Grafana",
|
||||
"client_url": "",
|
||||
"dedup_key": "alertId-0",
|
||||
"dedup_key": "key-" + strings.Repeat("x", 250),
|
||||
"event_action": "trigger",
|
||||
"links": []interface{}{
|
||||
map[string]interface{}{
|
||||
@@ -297,6 +299,7 @@ func TestPagerdutyNotifier(t *testing.T) {
|
||||
"class": "aClass",
|
||||
"component": "aComponent",
|
||||
"severity": "warning",
|
||||
"dedup_key": "key-" + strings.Repeat("x", 250),
|
||||
"keyOnly": "",
|
||||
},
|
||||
"severity": "warning",
|
||||
|
||||
Reference in New Issue
Block a user