Apply templating on alert notifications on OK state (#47355)

* OK notifications using previous evaluation data

* copy rule.EvalMatches to avoid changes to the underlying array

* test cases added/modified

* delete trailing newline

* fix double newline in go import

* add change to the changelog

* specify that this only affects legacy alerting (changelog)

* use current eval data instead of prev eval data

* create evalMatch just once

* code comments, renamings, getTemplateMatches() function

* changelog and docs updated

pkg/services/alerting/conditions/query.go

@@ -56,7 +56,11 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 
 	emptySeriesCount := 0
 	evalMatchCount := 0
+
+	// matches represents all the series that violate the alert condition
 	var matches []*alerting.EvalMatch
+	// allMatches capture all evaluation matches irregardless on whether the condition is met or not
+	var allMatches []*alerting.EvalMatch
 
 	for _, series := range seriesList {
 		reducedValue := c.Reducer.Reduce(series)
@@ -72,14 +76,17 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 			})
 		}
 
+		em := alerting.EvalMatch{
+			Metric: series.Name,
+			Value:  reducedValue,
+			Tags:   series.Tags,
+		}
+
+		allMatches = append(allMatches, &em)
+
 		if evalMatch {
 			evalMatchCount++
-
-			matches = append(matches, &alerting.EvalMatch{
-				Metric: series.Name,
-				Value:  reducedValue,
-				Tags:   series.Tags,
-			})
+			matches = append(matches, &em)
 		}
 	}
 
@@ -105,6 +112,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 		NoDataFound: emptySeriesCount == len(seriesList),
 		Operator:    c.Operator,
 		EvalMatches: matches,
+		AllMatches:  allMatches,
 	}, nil
 }
 

pkg/services/alerting/eval_context.go

@@ -17,6 +17,7 @@ type EvalContext struct {
 	IsTestRun      bool
 	IsDebug        bool
 	EvalMatches    []*EvalMatch
+	AllMatches     []*EvalMatch
 	Logs           []*ResultLogEntry
 	Error          error
 	ConditionEvals string
@@ -47,6 +48,7 @@ func NewEvalContext(alertCtx context.Context, rule *Rule, requestValidator model
 		Rule:             rule,
 		Logs:             make([]*ResultLogEntry, 0),
 		EvalMatches:      make([]*EvalMatch, 0),
+		AllMatches:       make([]*EvalMatch, 0),
 		Log:              log.New("alerting.evalContext"),
 		PrevAlertState:   rule.State,
 		RequestValidator: requestValidator,
@@ -188,11 +190,13 @@ func getNewStateInternal(c *EvalContext) models.AlertStateType {
 // evaluateNotificationTemplateFields will treat the alert evaluation rule's name and message fields as
 // templates, and evaluate the templates using data from the alert evaluation's tags
 func (c *EvalContext) evaluateNotificationTemplateFields() error {
-	if len(c.EvalMatches) < 1 {
+	matches := c.getTemplateMatches()
+	if len(matches) < 1 {
+		// if there are no series to parse the templates with, return
 		return nil
 	}
 
-	templateDataMap, err := buildTemplateDataMap(c.EvalMatches)
+	templateDataMap, err := buildTemplateDataMap(matches)
 	if err != nil {
 		return err
 	}
@@ -212,6 +216,18 @@ func (c *EvalContext) evaluateNotificationTemplateFields() error {
 	return nil
 }
 
+// getTemplateMatches returns the values we should use to parse the templates
+func (c *EvalContext) getTemplateMatches() []*EvalMatch {
+	// EvalMatches represent series violating the rule threshold,
+	// if we have any, this means the alert is firing and we should use this data to parse the templates.
+	if len(c.EvalMatches) > 0 {
+		return c.EvalMatches
+	}
+
+	// If we don't have any alerting values, use all values to parse the templates.
+	return c.AllMatches
+}
+
 func evaluateTemplate(s string, m map[string]string) (string, error) {
 	for k, v := range m {
 		re, err := regexp.Compile(fmt.Sprintf(`\${%s}`, regexp.QuoteMeta(k)))

pkg/services/alerting/eval_context_test.go

@@ -340,3 +340,66 @@ func TestEvaluateTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestEvaluateNotificationTemplateFields(t *testing.T) {
+	tests := []struct {
+		name            string
+		evalMatches     []*EvalMatch
+		allMatches      []*EvalMatch
+		expectedName    string
+		expectedMessage string
+	}{
+		{
+			"with evaluation matches",
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: test2",
+		},
+		{
+			"missing key",
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value3": "test2"},
+			}},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value3": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: ${value2}",
+		},
+		{
+			"no evaluation matches, with series",
+			[]*EvalMatch{},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: test2",
+		},
+		{
+			"no evaluation matches, no series",
+			[]*EvalMatch{},
+			[]*EvalMatch{},
+			"Rule name: ${value1}",
+			"Rule message: ${value2}",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(tt *testing.T) {
+			evalContext := NewEvalContext(context.Background(), &Rule{Name: "Rule name: ${value1}", Message: "Rule message: ${value2}", Conditions: []Condition{&conditionStub{firing: true}}}, &validations.OSSPluginRequestValidator{}, nil)
+			evalContext.EvalMatches = test.evalMatches
+			evalContext.AllMatches = test.allMatches
+
+			err := evalContext.evaluateNotificationTemplateFields()
+
+			require.NoError(tt, err)
+			require.Equal(tt, test.expectedName, evalContext.Rule.Name)
+			require.Equal(tt, test.expectedMessage, evalContext.Rule.Message)
+		})
+	}
+}

pkg/services/alerting/eval_handler.go

@@ -68,6 +68,7 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) {
 		}
 
 		context.EvalMatches = append(context.EvalMatches, cr.EvalMatches...)
+		context.AllMatches = append(context.AllMatches, cr.AllMatches...)
 	}
 
 	context.ConditionEvals = conditionEvals + " = " + strconv.FormatBool(firing)

pkg/services/alerting/interfaces.go

@@ -56,6 +56,7 @@ type ConditionResult struct {
 	NoDataFound bool
 	Operator    string
 	EvalMatches []*EvalMatch
+	AllMatches  []*EvalMatch
 }
 
 // Condition is responsible for evaluating an alert condition.

pkg/services/alerting/notifier_test.go

@@ -25,6 +25,7 @@ func TestNotificationService(t *testing.T) {
 	evalCtx := NewEvalContext(context.Background(), testRule, &validations.OSSPluginRequestValidator{}, store)
 
 	testRuleTemplated := &Rule{Name: "Test latency ${quantile}", Message: "Something is bad on instance ${instance}"}
+
 	evalCtxWithMatch := NewEvalContext(context.Background(), testRuleTemplated, &validations.OSSPluginRequestValidator{}, store)
 	evalCtxWithMatch.EvalMatches = []*EvalMatch{{
 		Tags: map[string]string{

pkg/services/ngalert/CHANGELOG.md

@@ -46,5 +46,6 @@ Scopes must have an order to ensure consistency and ease of search, this helps u
 ## Grafana Alerting - main / unreleased
 
 - [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216
+- [BUGFIX] (Legacy) Templates: Parse notification templates using all the matches of the alert rule when going from `Alerting` to `OK` in legacy alerting #47355
 - [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670 
 - [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971