Alerting: Support values in notification templates (#56457)

We have received a lot of feedback regarding the ValueString in alert notifications. Perhaps one of the most frequent complaints about ValueString is that it is difficult to read because it contains a lot of information, and the information is shown as a JSON-like string. Users have often asked how it can be templated and the answer is that it can't.

Until now users have been able to add custom annotations to their alert rules which contains values via the $values variable added in previous versions of Grafana. However, these custom annotations must be added for each of the user's alert rule, instead of once in a template that all of their alerts can be notified via.

This commit adds then the much requested feature to support values in notification templates. Users can then create a single template that prints the annotations, labels and values of their alerts in a format of their choice!

pkg/services/ngalert/notifier/channels/default_template.go

@@ -16,8 +16,12 @@ const (
 var DefaultTemplateString = `
 {{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
 
+{{ define "__text_values_list" }}{{ $len := len .Values }}{{ if $len }}{{ $first := gt $len 1 }}{{ range $refID, $value := .Values -}}
+{{ $refID }}={{ $value }}{{ if $first }}, {{ end }}{{ $first = false }}{{ end -}}
+{{ else }}[no value]{{ end }}{{ end }}
+
 {{ define "__text_alert_list" }}{{ range . }}
-Value: {{ or .ValueString "[no value]" }}
+Value: {{ template "__text_values_list" . }}
 Labels:
 {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
 {{ end }}Annotations:
@@ -38,7 +42,7 @@ Labels:
 
 
 {{ define "__teams_text_alert_list" }}{{ range . }}
-Value: {{ or .ValueString "[no value]" }}
+Value: {{ template "__text_values_list" . }}
 Labels:
 {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
 {{ end }}
@@ -70,8 +74,12 @@ Annotations:
 const TemplateForTestsString = `
 {{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
 
+{{ define "__text_values_list" }}{{ $len := len .Values }}{{ if $len }}{{ $first := gt $len 1 }}{{ range $refID, $value := .Values -}}
+{{ $refID }}={{ $value }}{{ if $first }}, {{ end }}{{ $first = false }}{{ end -}}
+{{ else }}[no value]{{ end }}{{ end }}
+
 {{ define "__text_alert_list" }}{{ range . }}
-Value: {{ or .ValueString "[no value]" }}
+Value: {{ template "__text_values_list" . }}
 Labels:
 {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
 {{ end }}Annotations:

pkg/services/ngalert/notifier/channels/default_template_test.go

@@ -20,7 +20,7 @@ func TestDefaultTemplateString(t *testing.T) {
 			Alert: model.Alert{
 				Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
 				Annotations: model.LabelSet{
-					"ann1": "annv1", "__dashboardUid__": "dbuid123", "__panelId__": "puid123", "__value_string__": "1234",
+					"ann1": "annv1", "__dashboardUid__": "dbuid123", "__panelId__": "puid123", "__values__": "{\"A\": 1234}", "__value_string__": "1234",
 				},
 				StartsAt:     time.Now(),
 				EndsAt:       time.Now().Add(1 * time.Hour),
@@ -29,7 +29,7 @@ func TestDefaultTemplateString(t *testing.T) {
 		}, { // Firing without dashboard and panel ID.
 			Alert: model.Alert{
 				Labels:       model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
-				Annotations:  model.LabelSet{"ann1": "annv2", "__value_string__": "1234"},
+				Annotations:  model.LabelSet{"ann1": "annv2", "__values__": "{\"A\": 1234}", "__value_string__": "1234"},
 				StartsAt:     time.Now(),
 				EndsAt:       time.Now().Add(2 * time.Hour),
 				GeneratorURL: "http://localhost/alert2",
@@ -38,7 +38,7 @@ func TestDefaultTemplateString(t *testing.T) {
 			Alert: model.Alert{
 				Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val3"},
 				Annotations: model.LabelSet{
-					"ann1": "annv3", "__dashboardUid__": "dbuid456", "__panelId__": "puid456", "__value_string__": "1234",
+					"ann1": "annv3", "__dashboardUid__": "dbuid456", "__panelId__": "puid456", "__values__": "{\"A\": 1234}", "__value_string__": "1234",
 				},
 				StartsAt:     time.Now().Add(-1 * time.Hour),
 				EndsAt:       time.Now().Add(-30 * time.Minute),
@@ -47,7 +47,7 @@ func TestDefaultTemplateString(t *testing.T) {
 		}, { // Resolved without dashboard and panel ID.
 			Alert: model.Alert{
 				Labels:       model.LabelSet{"alertname": "alert1", "lbl1": "val4"},
-				Annotations:  model.LabelSet{"ann1": "annv4", "__value_string__": "1234"},
+				Annotations:  model.LabelSet{"ann1": "annv4", "__values__": "{\"A\": 1234}", "__value_string__": "1234"},
 				StartsAt:     time.Now().Add(-2 * time.Hour),
 				EndsAt:       time.Now().Add(-3 * time.Hour),
 				GeneratorURL: "http://localhost/alert4",
@@ -91,7 +91,7 @@ func TestDefaultTemplateString(t *testing.T) {
 			templateString: DefaultMessageEmbed,
 			expected: `**Firing**
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val1
@@ -102,7 +102,7 @@ Silence: http://localhost/grafana/alerting/silence/new?alertmanager=grafana&matc
 Dashboard: http://localhost/grafana/d/dbuid123
 Panel: http://localhost/grafana/d/dbuid123?viewPanel=puid123
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val2
@@ -114,7 +114,7 @@ Silence: http://localhost/grafana/alerting/silence/new?alertmanager=grafana&matc
 
 **Resolved**
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val3
@@ -125,7 +125,7 @@ Silence: http://localhost/grafana/alerting/silence/new?alertmanager=grafana&matc
 Dashboard: http://localhost/grafana/d/dbuid456
 Panel: http://localhost/grafana/d/dbuid456?viewPanel=puid456
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val4
@@ -139,7 +139,7 @@ Silence: http://localhost/grafana/alerting/silence/new?alertmanager=grafana&matc
 			templateString: `{{ template "teams.default.message" .}}`,
 			expected: `**Firing**
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val1
@@ -157,7 +157,7 @@ Panel: [http://localhost/grafana/d/dbuid123?viewPanel=puid123](http://localhost/
 
 
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val2
@@ -174,7 +174,7 @@ Silence: [http://localhost/grafana/alerting/silence/new?alertmanager=grafana&mat
 
 **Resolved**
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val3
@@ -192,7 +192,7 @@ Panel: [http://localhost/grafana/d/dbuid456?viewPanel=puid456](http://localhost/
 
 
 
-Value: 1234
+Value: A=1234
 Labels:
  - alertname = alert1
  - lbl1 = val4

pkg/services/ngalert/notifier/channels/dingding_test.go

@@ -36,7 +36,7 @@ func TestDingdingNotifier(t *testing.T) {
 				{
 					Alert: model.Alert{
 						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
-						Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__value_string__": "1234"},
+						Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__values__": "{\"A\": 1234}", "__value_string__": "1234"},
 					},
 				},
 			},
@@ -44,7 +44,7 @@ func TestDingdingNotifier(t *testing.T) {
 				"msgtype": "link",
 				"link": map[string]interface{}{
 					"messageUrl": "dingtalk://dingtalkclient/page/link?pc_slide=false&url=http%3A%2F%2Flocalhost%2Falerting%2Flist",
-					"text":       "**Firing**\n\nValue: 1234\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+					"text":       "**Firing**\n\nValue: A=1234\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
 					"title":      "[FIRING:1]  (val1)",
 				},
 			},

pkg/services/ngalert/notifier/channels/template_data.go

@@ -2,6 +2,7 @@ package channels
 
 import (
 	"context"
+	"encoding/json"
 	"net/url"
 	"path"
 	"sort"
@@ -18,19 +19,20 @@ import (
 )
 
 type ExtendedAlert struct {
-	Status        string      `json:"status"`
-	Labels        template.KV `json:"labels"`
-	Annotations   template.KV `json:"annotations"`
-	StartsAt      time.Time   `json:"startsAt"`
-	EndsAt        time.Time   `json:"endsAt"`
-	GeneratorURL  string      `json:"generatorURL"`
-	Fingerprint   string      `json:"fingerprint"`
-	SilenceURL    string      `json:"silenceURL"`
-	DashboardURL  string      `json:"dashboardURL"`
-	PanelURL      string      `json:"panelURL"`
-	ValueString   string      `json:"valueString"`
-	ImageURL      string      `json:"imageURL,omitempty"`
-	EmbeddedImage string      `json:"embeddedImage,omitempty"`
+	Status        string             `json:"status"`
+	Labels        template.KV        `json:"labels"`
+	Annotations   template.KV        `json:"annotations"`
+	StartsAt      time.Time          `json:"startsAt"`
+	EndsAt        time.Time          `json:"endsAt"`
+	GeneratorURL  string             `json:"generatorURL"`
+	Fingerprint   string             `json:"fingerprint"`
+	SilenceURL    string             `json:"silenceURL"`
+	DashboardURL  string             `json:"dashboardURL"`
+	PanelURL      string             `json:"panelURL"`
+	Values        map[string]float64 `json:"values"`
+	ValueString   string             `json:"valueString"` // TODO: Remove in Grafana 10
+	ImageURL      string             `json:"imageURL,omitempty"`
+	EmbeddedImage string             `json:"embeddedImage,omitempty"`
 }
 
 type ExtendedAlerts []ExtendedAlert
@@ -90,7 +92,13 @@ func extendAlert(alert template.Alert, externalURL string, logger log.Logger) *E
 	}
 
 	if alert.Annotations != nil {
-		extended.ValueString = alert.Annotations[`__value_string__`]
+		if s, ok := alert.Annotations[ngmodels.ValuesAnnotation]; ok {
+			if err := json.Unmarshal([]byte(s), &extended.Values); err != nil {
+				logger.Warn("failed to unmarshal values annotation", "err", err)
+			}
+		}
+		// TODO: Remove in Grafana 10
+		extended.ValueString = alert.Annotations[ngmodels.ValueStringAnnotation]
 	}
 
 	matchers := make([]string, 0)