From 456dac130337d6e147033b11f5a09a3fb3b3846b Mon Sep 17 00:00:00 2001 From: George Robinson <85952834+gerobinson@users.noreply.github.com> Date: Thu, 15 Jul 2021 13:10:56 +0100 Subject: [PATCH] Expand the value of math and reduce expressions in annotations and labels (#36611) * Expand the value of math and reduce expressions in annotations and labels This commit makes it possible to use the values of reduce and math expressions in annotations and labels via their RefIDs. It uses the Stringer interface to ensure that "{{ $values.A }}" still prints the value in decimal format while also making the labels for each RefID available with "{{ $values.A.Labels }}" and the float64 value with "{{ $values.A.Value }}" --- .../create-grafana-managed-rule.md | 9 ++++ pkg/services/ngalert/eval/eval.go | 8 ++- pkg/services/ngalert/eval/extract_md.go | 21 ++++++++ pkg/services/ngalert/eval/extract_md_test.go | 49 +++++++++++++++++ pkg/services/ngalert/state/cache.go | 54 +++++++++++++------ pkg/services/ngalert/state/cache_test.go | 34 ++++++++++++ pkg/services/ngalert/state/manager.go | 1 + pkg/services/ngalert/state/manager_test.go | 25 +++++++++ pkg/services/ngalert/state/state.go | 22 ++++++++ 9 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 pkg/services/ngalert/state/cache_test.go diff --git a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md index bf126fe63dc..6efe7bf3efc 100644 --- a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md +++ b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md @@ -103,6 +103,15 @@ Labels are key value pairs that categorize or identify an alert. Labels are use ![Details section](/static/img/docs/alerting/unified/rule-edit-details-8-0.png 'Details section screenshot') +#### Template variables + +The following template variables are available when expanding annotations and labels. + +/ Name / Description / +/ ------- / --------------- / +/ $labels / Labels contains the labels from the query or condition. For example, `{{ $labels.instance }}` and `{{ $labels.job }}`. / +/ $values / Values contains the values of all reduce and math expressions that were evaluated for this alert rule. For example, `{{ $values.A }}`, `{{ $values.A.Labels }}` and `{{ $values.A.Value }}` where `A` is the `refID` of the expression. / + ## Preview alerts To evaluate the rule and see what alerts it would produce, click **Preview alerts**. It will display a list of alerts with state and value for each one. diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index c7fd9cf2373..6f94a402bf8 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -68,10 +68,15 @@ type Result struct { EvaluatedAt time.Time EvaluationDuration time.Duration - // EvaluationSring is a string representation of evaluation data such + // EvaluationString is a string representation of evaluation data such // as EvalMatches (from "classic condition"), and in the future from operations // like SSE "math". EvaluationString string + + // Values contains the RefID and value of reduce and math expressions. + // It does not contain values for classic conditions as the values + // in classic conditions do not have a RefID. + Values map[string]NumberValueCapture } // State is an enum of the evaluation State for an alert instance. @@ -346,6 +351,7 @@ func evaluateExecutionResult(execResults ExecutionResults, ts time.Time) Results EvaluatedAt: ts, EvaluationDuration: time.Since(ts), EvaluationString: extractEvalString(f), + Values: extractValues(f), } switch { diff --git a/pkg/services/ngalert/eval/extract_md.go b/pkg/services/ngalert/eval/extract_md.go index fd515c74bcb..f908363bab2 100644 --- a/pkg/services/ngalert/eval/extract_md.go +++ b/pkg/services/ngalert/eval/extract_md.go @@ -65,3 +65,24 @@ func extractEvalString(frame *data.Frame) (s string) { return "" } + +// extractValues returns the RefID and value for all reduce and math expressions +// in the frame. It does not return values for classic conditions as the values +// in classic conditions do not have a RefID. It returns nil if there are +// no results in the frame. +func extractValues(frame *data.Frame) map[string]NumberValueCapture { + if frame == nil { + return nil + } + if frame.Meta == nil || frame.Meta.Custom == nil { + return nil + } + if caps, ok := frame.Meta.Custom.([]NumberValueCapture); ok { + v := make(map[string]NumberValueCapture, len(caps)) + for _, c := range caps { + v[c.Var] = c + } + return v + } + return nil +} diff --git a/pkg/services/ngalert/eval/extract_md_test.go b/pkg/services/ngalert/eval/extract_md_test.go index 5035df0d147..50a39817f2d 100644 --- a/pkg/services/ngalert/eval/extract_md_test.go +++ b/pkg/services/ngalert/eval/extract_md_test.go @@ -47,6 +47,55 @@ func TestExtractEvalString(t *testing.T) { } } +func TestExtractValues(t *testing.T) { + cases := []struct { + desc string + inFrame *data.Frame + values map[string]NumberValueCapture + }{{ + desc: "No values in frame returns nil", + inFrame: newMetaFrame(nil, ptr.Float64(1)), + values: nil, + }, { + desc: "Classic condition frame returns nil", + inFrame: newMetaFrame([]classic.EvalMatch{ + {Metric: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, ptr.Float64(1)), + values: nil, + }, { + desc: "Nil value", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: nil}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: nil}, + }, + }, { + desc: "1 value", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, + }, { + desc: "2 values", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + {Var: "B", Labels: nil, Value: ptr.Float64(2)}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + "B": {Var: "B", Value: ptr.Float64(2)}, + }, + }} + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.values, extractValues(tc.inFrame)) + }) + } +} + func newMetaFrame(custom interface{}, val *float64) *data.Frame { return data.NewFrame("", data.NewField("", nil, []*float64{val})). diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 8795082e131..89f370b4e00 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -3,6 +3,7 @@ package state import ( "bytes" "fmt" + "strconv" "strings" "sync" text_template "text/template" @@ -35,12 +36,10 @@ func (c *cache) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result) * c.mtxStates.Lock() defer c.mtxStates.Unlock() - templateData := make(map[string]string, len(result.Instance)+3) - for k, v := range result.Instance { - templateData[k] = v - } - attachRuleLabels(templateData, alertRule) - ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, templateData) + // clone the labels so we don't change eval.Result + labels := result.Instance.Copy() + attachRuleLabels(labels, alertRule) + ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, labels, result.Values) // if duplicate labels exist, alertRule label will take precedence lbs := mergeLabels(ruleLabels, result.Instance) @@ -89,11 +88,11 @@ func attachRuleLabels(m map[string]string, alertRule *ngModels.AlertRule) { m[prometheusModel.AlertNameLabel] = alertRule.Title } -func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, data map[string]string) (map[string]string, map[string]string) { +func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, labels map[string]string, values map[string]eval.NumberValueCapture) (map[string]string, map[string]string) { expand := func(original map[string]string) map[string]string { expanded := make(map[string]string, len(original)) for k, v := range original { - ev, err := expandTemplate(alertRule.Title, v, data) + ev, err := expandTemplate(alertRule.Title, v, labels, values) expanded[k] = ev if err != nil { c.log.Error("error in expanding template", "name", k, "value", v, "err", err.Error()) @@ -104,13 +103,28 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, da return expanded } - return expand(alertRule.Labels), expand(alertRule.Annotations) } -func expandTemplate(name, text string, data map[string]string) (result string, resultErr error) { +// templateCaptureValue represents each value in .Values in the annotations +// and labels template. +type templateCaptureValue struct { + Labels map[string]string + Value *float64 +} + +// String implements the Stringer interface to print the value of each RefID +// in the template via {{ $values.A }} rather than {{ $values.A.Value }}. +func (v templateCaptureValue) String() string { + if v.Value != nil { + return strconv.FormatFloat(*v.Value, 'f', -1, 64) + } + return "null" +} + +func expandTemplate(name, text string, labels map[string]string, values map[string]eval.NumberValueCapture) (result string, resultErr error) { name = "__alert_" + name - text = "{{- $labels := .Labels -}}" + text + text = "{{- $labels := .Labels -}}{{- $values := .Values -}}" + text // It'd better to have no alert description than to kill the whole process // if there's a bug in the template. defer func() { @@ -128,12 +142,22 @@ func expandTemplate(name, text string, data map[string]string) (result string, r return "", fmt.Errorf("error parsing template %v: %s", name, err.Error()) } var buffer bytes.Buffer - err = tmpl.Execute(&buffer, struct { + if err := tmpl.Execute(&buffer, struct { Labels map[string]string + Values map[string]templateCaptureValue }{ - Labels: data, - }) - if err != nil { + Labels: labels, + Values: func() map[string]templateCaptureValue { + m := make(map[string]templateCaptureValue) + for k, v := range values { + m[k] = templateCaptureValue{ + Labels: v.Labels, + Value: v.Value, + } + } + return m + }(), + }); err != nil { return "", fmt.Errorf("error executing template %v: %s", name, err.Error()) } return buffer.String(), nil diff --git a/pkg/services/ngalert/state/cache_test.go b/pkg/services/ngalert/state/cache_test.go new file mode 100644 index 00000000000..bb54b7823d3 --- /dev/null +++ b/pkg/services/ngalert/state/cache_test.go @@ -0,0 +1,34 @@ +package state + +import ( + "testing" + + "github.com/stretchr/testify/assert" + ptr "github.com/xorcare/pointer" +) + +func TestTemplateCaptureValueStringer(t *testing.T) { + cases := []struct { + name string + value templateCaptureValue + expected string + }{{ + name: "nil value returns null", + value: templateCaptureValue{Value: nil}, + expected: "null", + }, { + name: "1.0 is returned as integer value", + value: templateCaptureValue{Value: ptr.Float64(1.0)}, + expected: "1", + }, { + name: "1.1 is returned as decimal value", + value: templateCaptureValue{Value: ptr.Float64(1.1)}, + expected: "1.1", + }} + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, c.value.String()) + }) + } +} diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 406643be793..d3dfbe1886a 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -155,6 +155,7 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul EvaluationTime: result.EvaluatedAt, EvaluationState: result.State, EvaluationString: result.EvaluationString, + Values: NewEvaluationValues(result.Values), }) currentState.TrimResults(alertRule) oldState := currentState.State diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 80bb1021eb5..1e6ca5af369 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -70,6 +70,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, @@ -122,6 +123,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, @@ -144,6 +146,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime, @@ -200,10 +203,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime.Add(1 * time.Minute), @@ -258,10 +263,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(1 * time.Minute), @@ -327,14 +334,17 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(80 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(80 * time.Second), @@ -392,10 +402,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -453,10 +465,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime, @@ -514,10 +528,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -575,10 +591,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -636,10 +654,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -698,10 +718,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -760,10 +782,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Error, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -815,6 +839,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index d0071dd9b43..a622ee85e3a 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -28,6 +28,28 @@ type Evaluation struct { EvaluationTime time.Time EvaluationState eval.State EvaluationString string + // Values contains the RefID and value of reduce and math expressions. + // It does not contain values for classic conditions as the values + // in classic conditions do not have a RefID. + Values map[string]EvaluationValue +} + +// EvaluationValue contains the labels and value for a RefID in an evaluation. +type EvaluationValue struct { + Labels data.Labels + Value *float64 +} + +// NewEvaluationValues returns the labels and values for each RefID in the capture. +func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]EvaluationValue { + result := make(map[string]EvaluationValue, len(m)) + for k, v := range m { + result[k] = EvaluationValue{ + Labels: v.Labels, + Value: v.Value, + } + } + return result } func (a *State) resultNormal(alertRule *ngModels.AlertRule, result eval.Result) {