From 8dd3eb856de497b53b6065c533a181908286b6f1 Mon Sep 17 00:00:00 2001 From: George Robinson Date: Thu, 20 Jul 2023 09:07:00 +0100 Subject: [PATCH] Alerting: Improve performance of matching captures (#71828) This commit updates eval.go to improve the performance of matching captures in the general case. In some cases we have reduced the runtime of the function from 10s of minutes to a couple 100ms. In the case where no capture matches the exact labels, we revert to the current subset/superset match, but with a reduced search space due to grouping captures. --- pkg/services/ngalert/eval/eval.go | 40 +++++++++---- pkg/services/ngalert/eval/eval_bench_test.go | 59 ++++++++++++++++++++ 2 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 pkg/services/ngalert/eval/eval_bench_test.go diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index 77a3d5e6ae2..71a0aee28cd 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -302,14 +302,20 @@ type NumberValueCapture struct { } func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.QueryDataResponse) ExecutionResults { - // eval captures for the '__value_string__' annotation and the Value property of the API response. - captures := make([]NumberValueCapture, 0, len(execResp.Responses)) - captureVal := func(refID string, labels data.Labels, value *float64) { - captures = append(captures, NumberValueCapture{ + // captures contains the values of all instant queries and expressions for each dimension + captures := make(map[string]map[data.Fingerprint]NumberValueCapture) + captureFn := func(refID string, labels data.Labels, value *float64) { + m := captures[refID] + if m == nil { + m = make(map[data.Fingerprint]NumberValueCapture) + } + fp := labels.Fingerprint() + m[fp] = NumberValueCapture{ Var: refID, Value: value, Labels: labels.Copy(), - }) + } + captures[refID] = m } // datasourceUIDsForRefIDs is a short-lived lookup table of RefID to DatasourceUID @@ -357,7 +363,7 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q if frame.Fields[0].Len() == 1 { v = frame.At(0, 0).(*float64) // type checked above } - captureVal(frame.RefID, frame.Fields[0].Labels, v) + captureFn(frame.RefID, frame.Fields[0].Labels, v) } if refID == c.Condition { @@ -379,13 +385,27 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q if len(frame.Fields) == 1 { theseLabels := frame.Fields[0].Labels - for _, cap := range captures { - // matching labels are equal labels, or when one set of labels includes the labels of the other. - if theseLabels.Equals(cap.Labels) || theseLabels.Contains(cap.Labels) || cap.Labels.Contains(theseLabels) { + fp := theseLabels.Fingerprint() + + for _, fps := range captures { + // First look for a capture whose labels are an exact match + if v, ok := fps[fp]; ok { if frame.Meta.Custom == nil { frame.Meta.Custom = []NumberValueCapture{} } - frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), cap) + frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), v) + } else { + // If no exact match was found, look for captures whose labels are either subsets + // or supersets + for _, v := range fps { + // matching labels are equal labels, or when one set of labels includes the labels of the other. + if theseLabels.Equals(v.Labels) || theseLabels.Contains(v.Labels) || v.Labels.Contains(theseLabels) { + if frame.Meta.Custom == nil { + frame.Meta.Custom = []NumberValueCapture{} + } + frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), v) + } + } } } } diff --git a/pkg/services/ngalert/eval/eval_bench_test.go b/pkg/services/ngalert/eval/eval_bench_test.go new file mode 100644 index 00000000000..600120cbe96 --- /dev/null +++ b/pkg/services/ngalert/eval/eval_bench_test.go @@ -0,0 +1,59 @@ +package eval + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/expr" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/util" +) + +func BenchmarkEvaluate(b *testing.B) { + var dataResp backend.QueryDataResponse + seedDataResponse(&dataResp, 10000) + var evaluator ConditionEvaluator = &conditionEvaluator{ + expressionService: &fakeExpressionService{ + hook: func(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error) { + return &dataResp, nil + }, + }, + condition: models.Condition{ + Condition: "B", + }, + } + for i := 0; i < b.N; i++ { + _, err := evaluator.Evaluate(context.Background(), time.Now()) + if err != nil { + b.Fatalf("Unexpected error: %s", err) + } + } +} + +func seedDataResponse(r *backend.QueryDataResponse, n int) { + resps := make(backend.Responses, n) + for i := 0; i < n; i++ { + labels := data.Labels{ + "foo": strconv.Itoa(i), + "bar": strconv.Itoa(i + 1), + } + a, b := resps["A"], resps["B"] + a.Frames = append(a.Frames, &data.Frame{ + Fields: data.Fields{ + data.NewField("Time", labels, []time.Time{time.Now()}), + data.NewField("Value", labels, []*float64{util.Pointer(1.0)}), + }, + }) + b.Frames = append(b.Frames, &data.Frame{ + Fields: data.Fields{ + data.NewField("Value", labels, []*float64{util.Pointer(1.0)}), + }, + }) + resps["A"], resps["B"] = a, b + } + r.Responses = resps +}