SSE: Localize/Contain Errors within an Expression (#73163)

Changes SSE to not always fail all queries when one fails. Now only the query itself, and nodes that depend on it will error.
---------

Co-authored-by: Gilles De Mey <gilles.de.mey@gmail.com>

pkg/services/ngalert/eval/eval.go

@@ -23,6 +23,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util/errutil"
 )
 
 var logger = log.New("ngalert.eval")
@@ -134,6 +135,9 @@ type ExecutionResults struct {
 	// Results contains the results of all queries, reduce and math expressions
 	Results map[string]data.Frames
 
+	// Errors contains a map of RefIDs that returned an error
+	Errors map[string]error
+
 	// NoData contains the DatasourceUID for RefIDs that returned no data.
 	NoData map[string]string
 
@@ -323,6 +327,7 @@ type NumberValueCapture struct {
 	Value *float64
 }
 
+//nolint:gocyclo
 func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.QueryDataResponse) ExecutionResults {
 	// captures contains the values of all instant queries and expressions for each dimension
 	captures := make(map[string]map[data.Fingerprint]NumberValueCapture)
@@ -349,6 +354,16 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 
 	result := ExecutionResults{Results: make(map[string]data.Frames)}
 	for refID, res := range execResp.Responses {
+		if res.Error != nil {
+			if result.Errors == nil {
+				result.Errors = make(map[string]error)
+			}
+			result.Errors[refID] = res.Error
+			if refID == c.Condition {
+				result.Error = res.Error
+			}
+		}
+
 		// There are two possible frame formats for No Data:
 		//
 		// 1. A response with no frames
@@ -431,6 +446,29 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 		}
 	}
 
+	// If the error of the condition is an Error that indicates the condition failed
+	// because one of its dependent query or expressions failed, then we follow
+	// the dependency chain to an error that is not a dependency error.
+	if len(result.Errors) > 0 && result.Error != nil {
+		if errors.Is(result.Error, expr.DependencyError) {
+			var utilError errutil.Error
+			e := result.Error
+			for {
+				errors.As(e, &utilError)
+				depRefID := utilError.PublicPayload["depRefId"].(string)
+				depError, ok := result.Errors[depRefID]
+				if !ok {
+					return result
+				}
+				if !errors.Is(depError, expr.DependencyError) {
+					result.Error = depError
+					return result
+				}
+				e = depError
+			}
+		}
+	}
+
 	return result
 }