From ba15d675e7c98f83288f13df1b0fc3703c67b728 Mon Sep 17 00:00:00 2001 From: Alex Moreno Date: Thu, 3 Nov 2022 10:35:34 +0100 Subject: [PATCH] Alerting: Add values to annotations (#57738) * Add values to annotations * Fix imports * Use State attrs instead of Result attrs * Remove unnecessary variable --- .../ngalert/state/historian/annotation.go | 44 +++++++++++++++++-- pkg/services/ngalert/state/manager.go | 6 ++- pkg/services/ngalert/state/manager_test.go | 10 ++++- pkg/services/ngalert/state/persist.go | 3 +- pkg/services/ngalert/state/testing.go | 3 +- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/pkg/services/ngalert/state/historian/annotation.go b/pkg/services/ngalert/state/historian/annotation.go index a4cbb892b2d..c9f56338700 100644 --- a/pkg/services/ngalert/state/historian/annotation.go +++ b/pkg/services/ngalert/state/historian/annotation.go @@ -3,14 +3,17 @@ package historian import ( "context" "fmt" + "sort" "strconv" "strings" "time" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/ngalert/eval" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" ) @@ -30,19 +33,18 @@ func NewAnnotationHistorian(annotations annotations.Repository, dashboards dashb } } -func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmodels.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData state.InstanceStateAndReason) { +func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmodels.AlertRule, currentState *state.State, evaluatedAt time.Time, currentData, previousData state.InstanceStateAndReason) { logger := h.log.New(rule.GetKey().LogContext()...) logger.Debug("Alert state changed creating annotation", "newState", currentData.String(), "oldState", previousData.String()) - labels = removePrivateLabels(labels) - annotationText := fmt.Sprintf("%s {%s} - %s", rule.Title, labels.String(), currentData.String()) - + annotationText, annotationData := buildAnnotationTextAndData(rule, currentState) item := &annotations.Item{ AlertId: rule.ID, OrgId: rule.OrgID, PrevState: previousData.String(), NewState: currentData.String(), Text: annotationText, + Data: annotationData, Epoch: evaluatedAt.UnixNano() / int64(time.Millisecond), } @@ -72,6 +74,40 @@ func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmode } } +func buildAnnotationTextAndData(rule *ngmodels.AlertRule, currentState *state.State) (string, *simplejson.Json) { + jsonData := simplejson.New() + var value string + + switch currentState.State { + case eval.Error: + if currentState.Error == nil { + jsonData.Set("error", nil) + } else { + jsonData.Set("error", currentState.Error.Error()) + } + value = "Error" + case eval.NoData: + jsonData.Set("noData", true) + value = "No data" + default: + keys := make([]string, 0, len(currentState.Values)) + for k := range currentState.Values { + keys = append(keys, k) + } + sort.Strings(keys) + + var values []string + for _, k := range keys { + values = append(values, fmt.Sprintf("%s=%f", k, currentState.Values[k])) + } + jsonData.Set("values", simplejson.NewFromAny(currentState.Values)) + value = strings.Join(values, ", ") + } + + labels := removePrivateLabels(currentState.Labels) + return fmt.Sprintf("%s {%s} - %s", rule.Title, labels.String(), value), jsonData +} + func removePrivateLabels(labels data.Labels) data.Labels { result := make(data.Labels) for k, v := range labels { diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 19cc5422dd3..900049819d4 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -243,7 +243,9 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason if shouldUpdateAnnotation && st.historian != nil { - go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason}) + go st.historian.RecordState(ctx, alertRule, currentState, result.EvaluatedAt, + InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, + InstanceStateAndReason{State: oldState, Reason: oldReason}) } return currentState } @@ -387,7 +389,7 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim s.EndsAt = evaluatedAt s.Resolved = true if st.historian != nil { - st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt, + st.historian.RecordState(ctx, alertRule, s, evaluatedAt, InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason}, previousState, ) diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 39a96545f1a..176fcd75111 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -49,15 +49,21 @@ func TestDashboardAnnotations(t *testing.T) { }) st.Warm(ctx) + bValue := float64(42) + cValue := float64(1) _ = st.ProcessEvalResults(ctx, evaluationTime, rule, eval.Results{{ Instance: data.Labels{"instance_label": "testValue2"}, State: eval.Alerting, EvaluatedAt: evaluationTime, + Values: map[string]eval.NumberValueCapture{ + "B": {Var: "B", Value: &bValue, Labels: data.Labels{"job": "prometheus"}}, + "C": {Var: "C", Value: &cValue, Labels: data.Labels{"job": "prometheus"}}, + }, }}, data.Labels{ "alertname": rule.Title, }) - expected := []string{rule.Title + " {alertname=" + rule.Title + ", instance_label=testValue2, test1=testValue1, test2=testValue2} - Alerting"} + expected := []string{rule.Title + " {alertname=" + rule.Title + ", instance_label=testValue2, test1=testValue1, test2=testValue2} - B=42.000000, C=1.000000"} sort.Strings(expected) require.Eventuallyf(t, func() bool { var actual []string @@ -1316,6 +1322,7 @@ func TestProcessEvalResults(t *testing.T) { eval.Result{ Instance: data.Labels{"instance_label": "test"}, State: eval.Error, + Error: errors.New("test error"), EvaluatedAt: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, }, @@ -1337,6 +1344,7 @@ func TestProcessEvalResults(t *testing.T) { Values: make(map[string]float64), State: eval.Pending, StateReason: eval.Error.String(), + Error: errors.New("test error"), Results: []state.Evaluation{ { EvaluationTime: evaluationTime, diff --git a/pkg/services/ngalert/state/persist.go b/pkg/services/ngalert/state/persist.go index 7fbf7a78776..db132d3b96b 100644 --- a/pkg/services/ngalert/state/persist.go +++ b/pkg/services/ngalert/state/persist.go @@ -4,7 +4,6 @@ import ( "context" "time" - "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/services/ngalert/models" ) @@ -24,5 +23,5 @@ type RuleReader interface { // Historian maintains an audit log of alert state history. type Historian interface { - RecordState(ctx context.Context, rule *models.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) + RecordState(ctx context.Context, rule *models.AlertRule, currentState *State, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) } diff --git a/pkg/services/ngalert/state/testing.go b/pkg/services/ngalert/state/testing.go index 080af67cf5a..b13029b3967 100644 --- a/pkg/services/ngalert/state/testing.go +++ b/pkg/services/ngalert/state/testing.go @@ -5,7 +5,6 @@ import ( "sync" "time" - "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/services/ngalert/models" ) @@ -50,5 +49,5 @@ func (f *FakeRuleReader) ListAlertRules(_ context.Context, q *models.ListAlertRu type FakeHistorian struct{} -func (f *FakeHistorian) RecordState(ctx context.Context, rule *models.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) { +func (f *FakeHistorian) RecordState(ctx context.Context, rule *models.AlertRule, currentState *State, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) { }