From 9dc422150893c9f1bd1819443f1dd75253e33565 Mon Sep 17 00:00:00 2001
From: Yuri Tseretyan <yuriy.tseretyan@grafana.com>
Date: Tue, 19 Mar 2024 10:00:03 -0400
Subject: [PATCH] Alerting: Log expression command types during evaluation
 (#84614)

---
 pkg/expr/classic/classic.go       |  4 +++
 pkg/expr/commands.go              | 15 +++++++++
 pkg/expr/graph.go                 | 52 +++++++++++++++++++++++++++++++
 pkg/expr/hysteresis.go            |  4 +++
 pkg/expr/ml/node.go               |  2 ++
 pkg/expr/ml/outlier.go            |  4 +++
 pkg/expr/ml/testing.go            |  4 +++
 pkg/expr/nodes.go                 |  7 +++++
 pkg/expr/sql_command.go           |  4 +++
 pkg/expr/threshold.go             |  4 +++
 pkg/services/ngalert/eval/eval.go |  1 +
 11 files changed, 101 insertions(+)

diff --git a/pkg/expr/classic/classic.go b/pkg/expr/classic/classic.go
index bcb99b24d7d..edc31d47b37 100644
--- a/pkg/expr/classic/classic.go
+++ b/pkg/expr/classic/classic.go
@@ -216,6 +216,10 @@ func (cmd *ConditionsCmd) executeCond(_ context.Context, _ time.Time, cond condi
 	return isCondFiring, isCondNoData, matches, nil
 }
 
+func (cmd *ConditionsCmd) Type() string {
+	return "classic_condition"
+}
+
 func compareWithOperator(b1, b2 bool, operator ConditionOperatorType) bool {
 	if operator == "or" {
 		return b1 || b2
diff --git a/pkg/expr/commands.go b/pkg/expr/commands.go
index 9e1c4958597..cc8f1f9fda6 100644
--- a/pkg/expr/commands.go
+++ b/pkg/expr/commands.go
@@ -19,6 +19,7 @@ import (
 type Command interface {
 	NeedsVars() []string
 	Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error)
+	Type() string
 }
 
 // MathCommand is a command for a math expression such as "1 + $GA / 2"
@@ -75,6 +76,10 @@ func (gm *MathCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.Va
 	return gm.Expression.Execute(gm.refID, vars, tracer)
 }
 
+func (gm *MathCommand) Type() string {
+	return TypeMath.String()
+}
+
 // ReduceCommand is an expression command for reduction of a timeseries such as a min, mean, or max.
 type ReduceCommand struct {
 	Reducer      mathexp.ReducerID
@@ -201,6 +206,10 @@ func (gr *ReduceCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.
 	return newRes, nil
 }
 
+func (gr *ReduceCommand) Type() string {
+	return TypeReduce.String()
+}
+
 // ResampleCommand is an expression command for resampling of a timeseries.
 type ResampleCommand struct {
 	Window        time.Duration
@@ -312,6 +321,10 @@ func (gr *ResampleCommand) Execute(ctx context.Context, now time.Time, vars math
 	return newRes, nil
 }
 
+func (gr *ResampleCommand) Type() string {
+	return TypeResample.String()
+}
+
 // CommandType is the type of the expression command.
 type CommandType int
 
@@ -342,6 +355,8 @@ func (gt CommandType) String() string {
 		return "resample"
 	case TypeClassicConditions:
 		return "classic_conditions"
+	case TypeThreshold:
+		return "threshold"
 	case TypeSQL:
 		return "sql"
 	default:
diff --git a/pkg/expr/graph.go b/pkg/expr/graph.go
index 49c90b6221b..d3d25e8e13a 100644
--- a/pkg/expr/graph.go
+++ b/pkg/expr/graph.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"slices"
 	"time"
 
 	"go.opentelemetry.io/otel/attribute"
+	"golang.org/x/exp/maps"
 	"gonum.org/v1/gonum/graph/simple"
 	"gonum.org/v1/gonum/graph/topo"
 
@@ -123,6 +125,56 @@ func (dp *DataPipeline) execute(c context.Context, now time.Time, s *Service) (m
 	return vars, nil
 }
 
+// GetDatasourceTypes returns an unique list of data source types used in the query. Machine learning node is encoded as `ml_<type>`, e.g. ml_outlier
+func (dp *DataPipeline) GetDatasourceTypes() []string {
+	if dp == nil {
+		return nil
+	}
+	m := make(map[string]struct{}, 2)
+	for _, node := range *dp {
+		name := ""
+		switch t := node.(type) {
+		case *DSNode:
+			if t.datasource != nil {
+				name = t.datasource.Type
+			}
+		case *MLNode:
+			name = fmt.Sprintf("ml_%s", t.command.Type())
+		}
+		if name == "" {
+			continue
+		}
+		m[name] = struct{}{}
+	}
+	result := maps.Keys(m)
+	slices.Sort(result)
+	return result
+}
+
+// GetCommandTypes returns a sorted unique list of all server-side expression commands used in the pipeline.
+func (dp *DataPipeline) GetCommandTypes() []string {
+	if dp == nil {
+		return nil
+	}
+	m := make(map[string]struct{}, 5) // 5 is big enough to cover most of the cases
+	for _, node := range *dp {
+		name := ""
+		switch t := node.(type) {
+		case *CMDNode:
+			if t.Command != nil {
+				name = t.Command.Type()
+			}
+		}
+		if name == "" {
+			continue
+		}
+		m[name] = struct{}{}
+	}
+	result := maps.Keys(m)
+	slices.Sort(result)
+	return result
+}
+
 // BuildPipeline builds a graph of the nodes, and returns the nodes in an
 // executable order.
 func (s *Service) buildPipeline(req *Request) (DataPipeline, error) {
diff --git a/pkg/expr/hysteresis.go b/pkg/expr/hysteresis.go
index 51ddcf37d5b..9f25653fca5 100644
--- a/pkg/expr/hysteresis.go
+++ b/pkg/expr/hysteresis.go
@@ -77,6 +77,10 @@ func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mat
 	return mathexp.Results{Values: append(loadingResults.Values, unloadingResults.Values...)}, nil
 }
 
+func (h HysteresisCommand) Type() string {
+	return "hysteresis"
+}
+
 func NewHysteresisCommand(refID string, referenceVar string, loadCondition ThresholdCommand, unloadCondition ThresholdCommand, l Fingerprints) (*HysteresisCommand, error) {
 	return &HysteresisCommand{
 		RefID:                  refID,
diff --git a/pkg/expr/ml/node.go b/pkg/expr/ml/node.go
index 771a19e4644..790c9375ac3 100644
--- a/pkg/expr/ml/node.go
+++ b/pkg/expr/ml/node.go
@@ -31,6 +31,8 @@ type Command interface {
 	// Execute creates a payload send request to the ML API by calling the function argument sendRequest, and then parses response.
 	// Function sendRequest is supposed to abstract the client configuration such creating http request, adding authorization parameters, host etc.
 	Execute(from, to time.Time, sendRequest func(method string, path string, payload []byte) (response.Response, error)) (*backend.QueryDataResponse, error)
+
+	Type() string
 }
 
 // UnmarshalCommand parses a config parameters and creates a command. Requires key `type` to be specified.
diff --git a/pkg/expr/ml/outlier.go b/pkg/expr/ml/outlier.go
index 40a53446c26..0cac000cc55 100644
--- a/pkg/expr/ml/outlier.go
+++ b/pkg/expr/ml/outlier.go
@@ -19,6 +19,10 @@ type OutlierCommand struct {
 
 var _ Command = OutlierCommand{}
 
+func (c OutlierCommand) Type() string {
+	return "outlier"
+}
+
 func (c OutlierCommand) DatasourceUID() string {
 	return c.config.DatasourceUID
 }
diff --git a/pkg/expr/ml/testing.go b/pkg/expr/ml/testing.go
index b10eeffe1a8..07c689b8ffb 100644
--- a/pkg/expr/ml/testing.go
+++ b/pkg/expr/ml/testing.go
@@ -42,3 +42,7 @@ func (f *FakeCommand) Execute(from, to time.Time, executor func(method string, p
 	}
 	return f.Response, f.Error
 }
+
+func (f *FakeCommand) Type() string {
+	return "fake"
+}
diff --git a/pkg/expr/nodes.go b/pkg/expr/nodes.go
index 14ac0fbe474..2d0bbbf0d2b 100644
--- a/pkg/expr/nodes.go
+++ b/pkg/expr/nodes.go
@@ -187,6 +187,13 @@ type DSNode struct {
 	request    Request
 }
 
+func (dn *DSNode) String() string {
+	if dn.datasource == nil {
+		return "unknown"
+	}
+	return dn.datasource.Type
+}
+
 // NodeType returns the data pipeline node type.
 func (dn *DSNode) NodeType() NodeType {
 	return TypeDatasourceNode
diff --git a/pkg/expr/sql_command.go b/pkg/expr/sql_command.go
index 44bbe559862..ec041e9abd6 100644
--- a/pkg/expr/sql_command.go
+++ b/pkg/expr/sql_command.go
@@ -103,3 +103,7 @@ func (gr *SQLCommand) Execute(ctx context.Context, now time.Time, vars mathexp.V
 
 	return rsp, nil
 }
+
+func (gr *SQLCommand) Type() string {
+	return TypeSQL.String()
+}
diff --git a/pkg/expr/threshold.go b/pkg/expr/threshold.go
index 8a8f9ea5417..dd840fd49a4 100644
--- a/pkg/expr/threshold.go
+++ b/pkg/expr/threshold.go
@@ -128,6 +128,10 @@ func (tc *ThresholdCommand) Execute(ctx context.Context, now time.Time, vars mat
 	return mathCommand.Execute(ctx, now, vars, tracer)
 }
 
+func (tc *ThresholdCommand) Type() string {
+	return TypeThreshold.String()
+}
+
 // createMathExpression converts all the info we have about a "threshold" expression in to a Math expression
 func createMathExpression(referenceVar string, thresholdFunc ThresholdType, args []float64, invert bool) (string, error) {
 	var exp string
diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go
index 29eab3ac08c..4a45ad2f513 100644
--- a/pkg/services/ngalert/eval/eval.go
+++ b/pkg/services/ngalert/eval/eval.go
@@ -73,6 +73,7 @@ func (r *conditionEvaluator) EvaluateRaw(ctx context.Context, now time.Time) (re
 		defer cancel()
 		execCtx = timeoutCtx
 	}
+	logger.FromContext(ctx).Debug("Executing pipeline", "commands", strings.Join(r.pipeline.GetCommandTypes(), ","), "datasources", strings.Join(r.pipeline.GetDatasourceTypes(), ","))
 	return r.expressionService.ExecutePipeline(execCtx, now, r.pipeline)
 }