diff --git a/docs/sources/alerting/old-alerting/add-notification-template.md b/docs/sources/alerting/old-alerting/add-notification-template.md index e9c276b5833..bd952bce72f 100644 --- a/docs/sources/alerting/old-alerting/add-notification-template.md +++ b/docs/sources/alerting/old-alerting/add-notification-template.md @@ -9,7 +9,9 @@ aliases = ["/docs/grafana/latest/alerting/add-notification-template/"] You can provide detailed information to alert notification recipients by injecting alert query data into an alert notification. This topic explains how you can use alert query labels in alert notifications. -Labels that exist from the evaluation of the alert query can be used in the alert rule name and in the alert notification message fields. The alert label data is injected into the notification fields when the alert is in the alerting state. When there are multiple unique values for the same label, the values are comma-separated. +Labels that exist from the evaluation of the alert query can be used in the alert rule name and in the alert notification message fields. The alert label data is injected into the notification fields when the alert is evaluated and the notification is about to be sent. When there are multiple unique values for the same label, the values are comma-separated. + +When the alert is in `firing` state, the data used to parse the templates corresponds to the series that are violating the alert condition at the moment the notification is sent. For `resolved` alert notifications, all series are used since there are none that are violating the condition. This topic explains how you can use alert query labels in alert notifications. diff --git a/pkg/services/alerting/conditions/query.go b/pkg/services/alerting/conditions/query.go index 9aee49e5b1b..4eb86714dd6 100644 --- a/pkg/services/alerting/conditions/query.go +++ b/pkg/services/alerting/conditions/query.go @@ -56,7 +56,11 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega emptySeriesCount := 0 evalMatchCount := 0 + + // matches represents all the series that violate the alert condition var matches []*alerting.EvalMatch + // allMatches capture all evaluation matches irregardless on whether the condition is met or not + var allMatches []*alerting.EvalMatch for _, series := range seriesList { reducedValue := c.Reducer.Reduce(series) @@ -72,14 +76,17 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega }) } + em := alerting.EvalMatch{ + Metric: series.Name, + Value: reducedValue, + Tags: series.Tags, + } + + allMatches = append(allMatches, &em) + if evalMatch { evalMatchCount++ - - matches = append(matches, &alerting.EvalMatch{ - Metric: series.Name, - Value: reducedValue, - Tags: series.Tags, - }) + matches = append(matches, &em) } } @@ -105,6 +112,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega NoDataFound: emptySeriesCount == len(seriesList), Operator: c.Operator, EvalMatches: matches, + AllMatches: allMatches, }, nil } diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index de0d2d155ab..7eb3caad471 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -17,6 +17,7 @@ type EvalContext struct { IsTestRun bool IsDebug bool EvalMatches []*EvalMatch + AllMatches []*EvalMatch Logs []*ResultLogEntry Error error ConditionEvals string @@ -47,6 +48,7 @@ func NewEvalContext(alertCtx context.Context, rule *Rule, requestValidator model Rule: rule, Logs: make([]*ResultLogEntry, 0), EvalMatches: make([]*EvalMatch, 0), + AllMatches: make([]*EvalMatch, 0), Log: log.New("alerting.evalContext"), PrevAlertState: rule.State, RequestValidator: requestValidator, @@ -188,11 +190,13 @@ func getNewStateInternal(c *EvalContext) models.AlertStateType { // evaluateNotificationTemplateFields will treat the alert evaluation rule's name and message fields as // templates, and evaluate the templates using data from the alert evaluation's tags func (c *EvalContext) evaluateNotificationTemplateFields() error { - if len(c.EvalMatches) < 1 { + matches := c.getTemplateMatches() + if len(matches) < 1 { + // if there are no series to parse the templates with, return return nil } - templateDataMap, err := buildTemplateDataMap(c.EvalMatches) + templateDataMap, err := buildTemplateDataMap(matches) if err != nil { return err } @@ -212,6 +216,18 @@ func (c *EvalContext) evaluateNotificationTemplateFields() error { return nil } +// getTemplateMatches returns the values we should use to parse the templates +func (c *EvalContext) getTemplateMatches() []*EvalMatch { + // EvalMatches represent series violating the rule threshold, + // if we have any, this means the alert is firing and we should use this data to parse the templates. + if len(c.EvalMatches) > 0 { + return c.EvalMatches + } + + // If we don't have any alerting values, use all values to parse the templates. + return c.AllMatches +} + func evaluateTemplate(s string, m map[string]string) (string, error) { for k, v := range m { re, err := regexp.Compile(fmt.Sprintf(`\${%s}`, regexp.QuoteMeta(k))) diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go index 30c3114ed15..6ac5da6fab0 100644 --- a/pkg/services/alerting/eval_context_test.go +++ b/pkg/services/alerting/eval_context_test.go @@ -340,3 +340,66 @@ func TestEvaluateTemplate(t *testing.T) { }) } } + +func TestEvaluateNotificationTemplateFields(t *testing.T) { + tests := []struct { + name string + evalMatches []*EvalMatch + allMatches []*EvalMatch + expectedName string + expectedMessage string + }{ + { + "with evaluation matches", + []*EvalMatch{{ + Tags: map[string]string{"value1": "test1", "value2": "test2"}, + }}, + []*EvalMatch{{ + Tags: map[string]string{"value1": "test1", "value2": "test2"}, + }}, + "Rule name: test1", + "Rule message: test2", + }, + { + "missing key", + []*EvalMatch{{ + Tags: map[string]string{"value1": "test1", "value3": "test2"}, + }}, + []*EvalMatch{{ + Tags: map[string]string{"value1": "test1", "value3": "test2"}, + }}, + "Rule name: test1", + "Rule message: ${value2}", + }, + { + "no evaluation matches, with series", + []*EvalMatch{}, + []*EvalMatch{{ + Tags: map[string]string{"value1": "test1", "value2": "test2"}, + }}, + "Rule name: test1", + "Rule message: test2", + }, + { + "no evaluation matches, no series", + []*EvalMatch{}, + []*EvalMatch{}, + "Rule name: ${value1}", + "Rule message: ${value2}", + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + evalContext := NewEvalContext(context.Background(), &Rule{Name: "Rule name: ${value1}", Message: "Rule message: ${value2}", Conditions: []Condition{&conditionStub{firing: true}}}, &validations.OSSPluginRequestValidator{}, nil) + evalContext.EvalMatches = test.evalMatches + evalContext.AllMatches = test.allMatches + + err := evalContext.evaluateNotificationTemplateFields() + + require.NoError(tt, err) + require.Equal(tt, test.expectedName, evalContext.Rule.Name) + require.Equal(tt, test.expectedMessage, evalContext.Rule.Message) + }) + } +} diff --git a/pkg/services/alerting/eval_handler.go b/pkg/services/alerting/eval_handler.go index 88a609a714b..f516d0718e6 100644 --- a/pkg/services/alerting/eval_handler.go +++ b/pkg/services/alerting/eval_handler.go @@ -68,6 +68,7 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) { } context.EvalMatches = append(context.EvalMatches, cr.EvalMatches...) + context.AllMatches = append(context.AllMatches, cr.AllMatches...) } context.ConditionEvals = conditionEvals + " = " + strconv.FormatBool(firing) diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 5d40116021b..b5f2fce922d 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -56,6 +56,7 @@ type ConditionResult struct { NoDataFound bool Operator string EvalMatches []*EvalMatch + AllMatches []*EvalMatch } // Condition is responsible for evaluating an alert condition. diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index c921908c6b8..4e1ce660709 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -25,6 +25,7 @@ func TestNotificationService(t *testing.T) { evalCtx := NewEvalContext(context.Background(), testRule, &validations.OSSPluginRequestValidator{}, store) testRuleTemplated := &Rule{Name: "Test latency ${quantile}", Message: "Something is bad on instance ${instance}"} + evalCtxWithMatch := NewEvalContext(context.Background(), testRuleTemplated, &validations.OSSPluginRequestValidator{}, store) evalCtxWithMatch.EvalMatches = []*EvalMatch{{ Tags: map[string]string{ diff --git a/pkg/services/ngalert/CHANGELOG.md b/pkg/services/ngalert/CHANGELOG.md index 5e1552efc72..c5650250638 100644 --- a/pkg/services/ngalert/CHANGELOG.md +++ b/pkg/services/ngalert/CHANGELOG.md @@ -46,5 +46,6 @@ Scopes must have an order to ensure consistency and ease of search, this helps u ## Grafana Alerting - main / unreleased - [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216 +- [BUGFIX] (Legacy) Templates: Parse notification templates using all the matches of the alert rule when going from `Alerting` to `OK` in legacy alerting #47355 - [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670 - [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971 \ No newline at end of file