diff --git a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md index bf126fe63dc..6efe7bf3efc 100644 --- a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md +++ b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md @@ -103,6 +103,15 @@ Labels are key value pairs that categorize or identify an alert. Labels are use ![Details section](/static/img/docs/alerting/unified/rule-edit-details-8-0.png 'Details section screenshot') +#### Template variables + +The following template variables are available when expanding annotations and labels. + +/ Name / Description / +/ ------- / --------------- / +/ $labels / Labels contains the labels from the query or condition. For example, `{{ $labels.instance }}` and `{{ $labels.job }}`. / +/ $values / Values contains the values of all reduce and math expressions that were evaluated for this alert rule. For example, `{{ $values.A }}`, `{{ $values.A.Labels }}` and `{{ $values.A.Value }}` where `A` is the `refID` of the expression. / + ## Preview alerts To evaluate the rule and see what alerts it would produce, click **Preview alerts**. It will display a list of alerts with state and value for each one. diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index c7fd9cf2373..6f94a402bf8 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -68,10 +68,15 @@ type Result struct { EvaluatedAt time.Time EvaluationDuration time.Duration - // EvaluationSring is a string representation of evaluation data such + // EvaluationString is a string representation of evaluation data such // as EvalMatches (from "classic condition"), and in the future from operations // like SSE "math". EvaluationString string + + // Values contains the RefID and value of reduce and math expressions. + // It does not contain values for classic conditions as the values + // in classic conditions do not have a RefID. + Values map[string]NumberValueCapture } // State is an enum of the evaluation State for an alert instance. @@ -346,6 +351,7 @@ func evaluateExecutionResult(execResults ExecutionResults, ts time.Time) Results EvaluatedAt: ts, EvaluationDuration: time.Since(ts), EvaluationString: extractEvalString(f), + Values: extractValues(f), } switch { diff --git a/pkg/services/ngalert/eval/extract_md.go b/pkg/services/ngalert/eval/extract_md.go index fd515c74bcb..f908363bab2 100644 --- a/pkg/services/ngalert/eval/extract_md.go +++ b/pkg/services/ngalert/eval/extract_md.go @@ -65,3 +65,24 @@ func extractEvalString(frame *data.Frame) (s string) { return "" } + +// extractValues returns the RefID and value for all reduce and math expressions +// in the frame. It does not return values for classic conditions as the values +// in classic conditions do not have a RefID. It returns nil if there are +// no results in the frame. +func extractValues(frame *data.Frame) map[string]NumberValueCapture { + if frame == nil { + return nil + } + if frame.Meta == nil || frame.Meta.Custom == nil { + return nil + } + if caps, ok := frame.Meta.Custom.([]NumberValueCapture); ok { + v := make(map[string]NumberValueCapture, len(caps)) + for _, c := range caps { + v[c.Var] = c + } + return v + } + return nil +} diff --git a/pkg/services/ngalert/eval/extract_md_test.go b/pkg/services/ngalert/eval/extract_md_test.go index 5035df0d147..50a39817f2d 100644 --- a/pkg/services/ngalert/eval/extract_md_test.go +++ b/pkg/services/ngalert/eval/extract_md_test.go @@ -47,6 +47,55 @@ func TestExtractEvalString(t *testing.T) { } } +func TestExtractValues(t *testing.T) { + cases := []struct { + desc string + inFrame *data.Frame + values map[string]NumberValueCapture + }{{ + desc: "No values in frame returns nil", + inFrame: newMetaFrame(nil, ptr.Float64(1)), + values: nil, + }, { + desc: "Classic condition frame returns nil", + inFrame: newMetaFrame([]classic.EvalMatch{ + {Metric: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, ptr.Float64(1)), + values: nil, + }, { + desc: "Nil value", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: nil}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: nil}, + }, + }, { + desc: "1 value", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, + }, { + desc: "2 values", + inFrame: newMetaFrame([]NumberValueCapture{ + {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + {Var: "B", Labels: nil, Value: ptr.Float64(2)}, + }, ptr.Float64(1)), + values: map[string]NumberValueCapture{ + "A": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + "B": {Var: "B", Value: ptr.Float64(2)}, + }, + }} + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.values, extractValues(tc.inFrame)) + }) + } +} + func newMetaFrame(custom interface{}, val *float64) *data.Frame { return data.NewFrame("", data.NewField("", nil, []*float64{val})). diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 8795082e131..89f370b4e00 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -3,6 +3,7 @@ package state import ( "bytes" "fmt" + "strconv" "strings" "sync" text_template "text/template" @@ -35,12 +36,10 @@ func (c *cache) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result) * c.mtxStates.Lock() defer c.mtxStates.Unlock() - templateData := make(map[string]string, len(result.Instance)+3) - for k, v := range result.Instance { - templateData[k] = v - } - attachRuleLabels(templateData, alertRule) - ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, templateData) + // clone the labels so we don't change eval.Result + labels := result.Instance.Copy() + attachRuleLabels(labels, alertRule) + ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, labels, result.Values) // if duplicate labels exist, alertRule label will take precedence lbs := mergeLabels(ruleLabels, result.Instance) @@ -89,11 +88,11 @@ func attachRuleLabels(m map[string]string, alertRule *ngModels.AlertRule) { m[prometheusModel.AlertNameLabel] = alertRule.Title } -func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, data map[string]string) (map[string]string, map[string]string) { +func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, labels map[string]string, values map[string]eval.NumberValueCapture) (map[string]string, map[string]string) { expand := func(original map[string]string) map[string]string { expanded := make(map[string]string, len(original)) for k, v := range original { - ev, err := expandTemplate(alertRule.Title, v, data) + ev, err := expandTemplate(alertRule.Title, v, labels, values) expanded[k] = ev if err != nil { c.log.Error("error in expanding template", "name", k, "value", v, "err", err.Error()) @@ -104,13 +103,28 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, da return expanded } - return expand(alertRule.Labels), expand(alertRule.Annotations) } -func expandTemplate(name, text string, data map[string]string) (result string, resultErr error) { +// templateCaptureValue represents each value in .Values in the annotations +// and labels template. +type templateCaptureValue struct { + Labels map[string]string + Value *float64 +} + +// String implements the Stringer interface to print the value of each RefID +// in the template via {{ $values.A }} rather than {{ $values.A.Value }}. +func (v templateCaptureValue) String() string { + if v.Value != nil { + return strconv.FormatFloat(*v.Value, 'f', -1, 64) + } + return "null" +} + +func expandTemplate(name, text string, labels map[string]string, values map[string]eval.NumberValueCapture) (result string, resultErr error) { name = "__alert_" + name - text = "{{- $labels := .Labels -}}" + text + text = "{{- $labels := .Labels -}}{{- $values := .Values -}}" + text // It'd better to have no alert description than to kill the whole process // if there's a bug in the template. defer func() { @@ -128,12 +142,22 @@ func expandTemplate(name, text string, data map[string]string) (result string, r return "", fmt.Errorf("error parsing template %v: %s", name, err.Error()) } var buffer bytes.Buffer - err = tmpl.Execute(&buffer, struct { + if err := tmpl.Execute(&buffer, struct { Labels map[string]string + Values map[string]templateCaptureValue }{ - Labels: data, - }) - if err != nil { + Labels: labels, + Values: func() map[string]templateCaptureValue { + m := make(map[string]templateCaptureValue) + for k, v := range values { + m[k] = templateCaptureValue{ + Labels: v.Labels, + Value: v.Value, + } + } + return m + }(), + }); err != nil { return "", fmt.Errorf("error executing template %v: %s", name, err.Error()) } return buffer.String(), nil diff --git a/pkg/services/ngalert/state/cache_test.go b/pkg/services/ngalert/state/cache_test.go new file mode 100644 index 00000000000..bb54b7823d3 --- /dev/null +++ b/pkg/services/ngalert/state/cache_test.go @@ -0,0 +1,34 @@ +package state + +import ( + "testing" + + "github.com/stretchr/testify/assert" + ptr "github.com/xorcare/pointer" +) + +func TestTemplateCaptureValueStringer(t *testing.T) { + cases := []struct { + name string + value templateCaptureValue + expected string + }{{ + name: "nil value returns null", + value: templateCaptureValue{Value: nil}, + expected: "null", + }, { + name: "1.0 is returned as integer value", + value: templateCaptureValue{Value: ptr.Float64(1.0)}, + expected: "1", + }, { + name: "1.1 is returned as decimal value", + value: templateCaptureValue{Value: ptr.Float64(1.1)}, + expected: "1.1", + }} + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, c.value.String()) + }) + } +} diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 406643be793..d3dfbe1886a 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -155,6 +155,7 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul EvaluationTime: result.EvaluatedAt, EvaluationState: result.State, EvaluationString: result.EvaluationString, + Values: NewEvaluationValues(result.Values), }) currentState.TrimResults(alertRule) oldState := currentState.State diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 80bb1021eb5..1e6ca5af369 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -70,6 +70,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, @@ -122,6 +123,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, @@ -144,6 +146,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime, @@ -200,10 +203,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime.Add(1 * time.Minute), @@ -258,10 +263,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(1 * time.Minute), @@ -327,14 +334,17 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(80 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(80 * time.Second), @@ -392,10 +402,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -453,10 +465,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime, @@ -514,10 +528,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -575,10 +591,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -636,10 +654,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -698,10 +718,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -760,10 +782,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Error, + Values: make(map[string]state.EvaluationValue), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -815,6 +839,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), }, }, LastEvaluationTime: evaluationTime, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index d0071dd9b43..a622ee85e3a 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -28,6 +28,28 @@ type Evaluation struct { EvaluationTime time.Time EvaluationState eval.State EvaluationString string + // Values contains the RefID and value of reduce and math expressions. + // It does not contain values for classic conditions as the values + // in classic conditions do not have a RefID. + Values map[string]EvaluationValue +} + +// EvaluationValue contains the labels and value for a RefID in an evaluation. +type EvaluationValue struct { + Labels data.Labels + Value *float64 +} + +// NewEvaluationValues returns the labels and values for each RefID in the capture. +func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]EvaluationValue { + result := make(map[string]EvaluationValue, len(m)) + for k, v := range m { + result[k] = EvaluationValue{ + Labels: v.Labels, + Value: v.Value, + } + } + return result } func (a *State) resultNormal(alertRule *ngModels.AlertRule, result eval.Result) {