diff --git a/docs/sources/alerting/add-notification-template.md b/docs/sources/alerting/add-notification-template.md new file mode 100644 index 00000000000..7a408e05ef7 --- /dev/null +++ b/docs/sources/alerting/add-notification-template.md @@ -0,0 +1,23 @@ ++++ +title = "Alert notification templating" +keywords = ["grafana", "documentation", "alerting", "alerts", "notification", "templating"] +weight = 110 ++++ + +# Alert notification templating + +You can provide detailed information to alert notification recipients by injecting alert query data into an alert notification. This topic explains how you can use alert query labels in alert notifications. + +Labels that exist from the evaluation of the alert query can be used in the alert rule name and in the alert notification message fields. The alert label data is injected into the notification fields when the alert is in the alerting state. When there are multiple unique values for the same label, the values are comma-separated. + +This topic explains how you can use alert query labels in alert notifications. + +## Adding alert label data into your alert notification + +1. Navigate to the panel you want to add or edit an alert rule for. +1. Click on the panel title, and then click **Edit**. +1. On the Alert tab, click **Create Alert**. If an alert already exists for this panel, then you can edit the alert directly. +1. Refer to the alert query labels in the alert rule name and/or alert notification message field by using the `${Label}` syntax. +1. Click **Save** in the upper right corner to save the alert rule and the dashboard. + +![Alerting notification template](/img/docs/alerting/notification_template.png) diff --git a/docs/sources/alerting/create-alerts.md b/docs/sources/alerting/create-alerts.md index 0206f3b2627..d986b1390d3 100644 --- a/docs/sources/alerting/create-alerts.md +++ b/docs/sources/alerting/create-alerts.md @@ -33,7 +33,7 @@ This section describes the fields you fill out to create an alert. ### Rule -- **Name -** Enter a descriptive name. The name will be displayed in the Alert Rules list. +- **Name -** Enter a descriptive name. The name will be displayed in the Alert Rules list. This field supports [templating]({{< relref "./add-notification-template.md" >}}). - **Evaluate every -** Specify how often the scheduler should evaluate the alert rule. This is referred to as the _evaluation interval_. - **For -** Specify how long the query needs to violate the configured thresholds before the alert notification triggers. @@ -117,7 +117,7 @@ The actual notifications are configured and shared between multiple alerts. Read [Alert notifications]({{< relref "notifications.md" >}}) for information on how to configure and set up notifications. - **Send to -** Select an alert notification channel if you have one set up. -- **Message -** Enter a text message to be sent on the notification channel. Some alert notifiers support transforming the text to HTML or other rich formats. +- **Message -** Enter a text message to be sent on the notification channel. Some alert notifiers support transforming the text to HTML or other rich formats. This field supports [templating]({{< relref "./add-notification-template.md" >}}). - **Tags -** Specify a list of tags (key/value) to be included in the notification. It is only supported by [some notifiers]({{< relref "notifications/#all-supported-notifiers" >}}). ## Alert state history and annotations diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 058b665fbf9..4dd8ca20c5d 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -230,3 +230,9 @@ Notification services which need public image access are marked as 'external onl All alert notifications contain a link back to the triggered alert in the Grafana instance. This URL is based on the [domain]({{< relref "../administration/configuration/#domain" >}}) setting in Grafana. + +## Notification templating + +> **Note:** Alert notification templating is only available in Grafana v7.4 and above. + +The alert notification template feature allows you to take the [label]({{< relref "../getting-started/timeseries-dimensions.md#labels" >}}) value from an alert query and [inject that into alert notifications]({{< relref "./add-notification-template.md" >}}). diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index a33bbe7181e..dc3c4bd4b71 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -3,6 +3,7 @@ package alerting import ( "context" "fmt" + "regexp" "time" "github.com/grafana/grafana/pkg/bus" @@ -178,3 +179,66 @@ func getNewStateInternal(c *EvalContext) models.AlertStateType { return models.AlertStateOK } + +// evaluateNotificationTemplateFields will treat the alert evaluation rule's name and message fields as +// templates, and evaluate the templates using data from the alert evaluation's tags +func (c *EvalContext) evaluateNotificationTemplateFields() error { + if len(c.EvalMatches) < 1 { + return nil + } + + templateDataMap, err := buildTemplateDataMap(c.EvalMatches) + if err != nil { + return err + } + + ruleMsg, err := evaluateTemplate(c.Rule.Message, templateDataMap) + if err != nil { + return err + } + c.Rule.Message = ruleMsg + + ruleName, err := evaluateTemplate(c.Rule.Name, templateDataMap) + if err != nil { + return err + } + c.Rule.Name = ruleName + + return nil +} + +func evaluateTemplate(s string, m map[string]string) (string, error) { + for k, v := range m { + re, err := regexp.Compile(fmt.Sprintf(`\${%s}`, regexp.QuoteMeta(k))) + if err != nil { + return "", err + } + s = re.ReplaceAllString(s, v) + } + + return s, nil +} + +// buildTemplateDataMap builds a map of alert evaluation tag names to a set of associated values (comma separated) +func buildTemplateDataMap(evalMatches []*EvalMatch) (map[string]string, error) { + var result = map[string]string{} + for _, match := range evalMatches { + for tagName, tagValue := range match.Tags { + // skip duplicate values + rVal, err := regexp.Compile(fmt.Sprintf(`\b%s\b`, regexp.QuoteMeta(tagValue))) + if err != nil { + return nil, err + } + rMatch := rVal.FindString(result[tagName]) + if len(rMatch) > 0 { + continue + } + if _, exists := result[tagName]; exists { + result[tagName] = fmt.Sprintf("%s, %s", result[tagName], tagValue) + } else { + result[tagName] = tagValue + } + } + } + return result, nil +} diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go index 76cbc90c640..14f0fba600d 100644 --- a/pkg/services/alerting/eval_context_test.go +++ b/pkg/services/alerting/eval_context_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/models" + "github.com/stretchr/testify/assert" ) func TestStateIsUpdatedWhenNeeded(t *testing.T) { @@ -204,3 +205,136 @@ func TestGetStateFromEvalContext(t *testing.T) { assert.Equal(t, tc.expected, newState, "failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(newState)) } } + +func TestBuildTemplateDataMap(t *testing.T) { + tcs := []struct { + name string + matches []*EvalMatch + expected map[string]string + }{ + { + name: "single match", + matches: []*EvalMatch{ + { + Tags: map[string]string{ + "InstanceId": "i-123456789", + "Percentile": "0.999", + }, + }, + }, + expected: map[string]string{ + "InstanceId": "i-123456789", + "Percentile": "0.999", + }, + }, + { + name: "matches with duplicate keys", + matches: []*EvalMatch{ + { + Tags: map[string]string{ + "InstanceId": "i-123456789", + }, + }, + { + Tags: map[string]string{ + "InstanceId": "i-987654321", + "Percentile": "0.999", + }, + }, + }, + expected: map[string]string{ + "InstanceId": "i-123456789, i-987654321", + "Percentile": "0.999", + }, + }, + { + name: "matches with duplicate keys and values", + matches: []*EvalMatch{ + { + Tags: map[string]string{ + "InstanceId": "i-123456789", + "Percentile": "0.999", + }, + }, + { + Tags: map[string]string{ + "InstanceId": "i-987654321", + "Percentile": "0.995", + }, + }, + { + Tags: map[string]string{ + "InstanceId": "i-987654321", + "Percentile": "0.999", + }, + }, + }, + expected: map[string]string{ + "InstanceId": "i-123456789, i-987654321", + "Percentile": "0.999, 0.995", + }, + }, + { + name: "a value and its substring for same key", + matches: []*EvalMatch{ + { + Tags: map[string]string{ + "Percentile": "0.9990", + }, + }, + { + Tags: map[string]string{ + "Percentile": "0.999", + }, + }, + }, + expected: map[string]string{ + "Percentile": "0.9990, 0.999", + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + result, err := buildTemplateDataMap(tc.matches) + require.NoError(t, err) + assert.Equal(t, tc.expected, result, "failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, result) + }) + } +} + +func TestEvaluateTemplate(t *testing.T) { + tcs := []struct { + name string + message string + data map[string]string + expected string + }{ + { + name: "matching terms", + message: "Degraded ${percentile} latency on ${instance}", + data: map[string]string{ + "instance": "i-123456789", + "percentile": "0.95", + }, + expected: "Degraded 0.95 latency on i-123456789", + }, + { + name: "non-matching terms", + message: "Degraded $percentile latency for endpoint ${ endpoint } on ${instance}", + data: map[string]string{ + "INSTANCE": "i-123456789", + "percentile": "0.95", + "endpoint": "/api/dashboard/123", + }, + expected: "Degraded $percentile latency for endpoint ${ endpoint } on ${instance}", + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + result, err := evaluateTemplate(tc.message, tc.data) + require.NoError(t, err) + assert.Equal(t, tc.expected, result, "failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, result) + }) + } +} diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 839b4eda027..238ef466685 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -131,9 +131,11 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no n.log.Debug("Sending notification", "type", notifier.GetType(), "uid", notifier.GetNotifierUID(), "isDefault", notifier.GetIsDefault()) metrics.MAlertingNotificationSent.WithLabelValues(notifier.GetType()).Inc() - err := notifier.Notify(evalContext) + if err := evalContext.evaluateNotificationTemplateFields(); err != nil { + n.log.Error("failed trying to evaluate notification template fields", "uid", notifier.GetNotifierUID(), "error", err) + } - if err != nil { + if err := notifier.Notify(evalContext); err != nil { n.log.Error("failed to send notification", "uid", notifier.GetNotifierUID(), "error", err) metrics.MAlertingNotificationFailed.WithLabelValues(notifier.GetType()).Inc() return err diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index fcb49e3abc0..78dcb86df13 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -6,9 +6,9 @@ import ( "time" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/assert" "github.com/grafana/grafana/pkg/bus" "github.com/stretchr/testify/require" @@ -18,18 +18,19 @@ import ( ) func TestNotificationService(t *testing.T) { - testRule := &Rule{ - ID: 1, - DashboardID: 1, - PanelID: 1, - OrgID: 1, - Name: "Test", - Message: "Something is bad", - State: models.AlertStateAlerting, - Notifications: []string{"1"}, - } + testRule := &Rule{Name: "Test", Message: "Something is bad"} evalCtx := NewEvalContext(context.Background(), testRule) + testRuleTemplated := &Rule{Name: "Test latency ${quantile}", Message: "Something is bad on instance ${instance}"} + evalCtxWithMatch := NewEvalContext(context.Background(), testRuleTemplated) + evalCtxWithMatch.EvalMatches = []*EvalMatch{{ + Tags: map[string]string{ + "instance": "localhost:3000", + "quantile": "0.99", + }, + }} + evalCtxWithoutMatch := NewEvalContext(context.Background(), testRuleTemplated) + notificationServiceScenario(t, "Given alert rule with upload image enabled should render and upload image and send notification", evalCtx, true, func(sc *scenarioContext) { err := sc.notificationService.SendIfNeeded(evalCtx) @@ -122,6 +123,32 @@ func TestNotificationService(t *testing.T) { require.Equalf(sc.t, 0, sc.imageUploadCount, "expected image not to be uploaded, but it was") require.Truef(sc.t, evalCtx.Ctx.Value(notificationSent{}).(bool), "expected notification to be sent, but wasn't") }) + + notificationServiceScenario(t, "Given matched alert rule with templated notification fields", + evalCtxWithMatch, true, func(sc *scenarioContext) { + err := sc.notificationService.SendIfNeeded(evalCtxWithMatch) + require.NoError(sc.t, err) + + ctx := evalCtxWithMatch + require.Equalf(sc.t, 1, sc.renderCount, "expected render to be called, but wasn't") + require.Equalf(sc.t, 1, sc.imageUploadCount, "expected image to be uploaded, but wasn't") + require.Truef(sc.t, ctx.Ctx.Value(notificationSent{}).(bool), "expected notification to be sent, but wasn't") + assert.Equal(t, "Test latency 0.99", ctx.Rule.Name) + assert.Equal(t, "Something is bad on instance localhost:3000", ctx.Rule.Message) + }) + + notificationServiceScenario(t, "Given unmatched alert rule with templated notification fields", + evalCtxWithoutMatch, true, func(sc *scenarioContext) { + err := sc.notificationService.SendIfNeeded(evalCtxWithMatch) + require.NoError(sc.t, err) + + ctx := evalCtxWithMatch + require.Equalf(sc.t, 1, sc.renderCount, "expected render to be called, but wasn't") + require.Equalf(sc.t, 1, sc.imageUploadCount, "expected image to be uploaded, but wasn't") + require.Truef(sc.t, ctx.Ctx.Value(notificationSent{}).(bool), "expected notification to be sent, but wasn't") + assert.Equal(t, evalCtxWithoutMatch.Rule.Name, ctx.Rule.Name) + assert.Equal(t, evalCtxWithoutMatch.Rule.Message, ctx.Rule.Message) + }) } type scenarioContext struct { diff --git a/pkg/tsdb/cloudmonitoring/time_series_filter.go b/pkg/tsdb/cloudmonitoring/time_series_filter.go index 152fca3c633..7a8e1c169fd 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_filter.go +++ b/pkg/tsdb/cloudmonitoring/time_series_filter.go @@ -258,6 +258,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) handleNonDistributionSe metricName := formatLegendKeys(series.Metric.Type, defaultMetricName, seriesLabels, nil, timeSeriesFilter) dataField := frame.Fields[1] dataField.Name = metricName + dataField.Labels = seriesLabels } func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseToAnnotations(queryRes *tsdb.QueryResult, data cloudMonitoringResponse, title string, text string, tags string) error { diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index e84d371a2d1..699e0754663 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -8,7 +8,10 @@
Name - + + + If you want to apply templating to the alert rule name, you must use the following syntax - ${Label} +
Evaluate every @@ -186,11 +189,14 @@
Message + + If you want to apply templating to the alert rule name, you must use the following syntax - ${Label} +
Tags