Alerting: Recording rules understands errors embedded in dataframes (#88946)

* Make MakeDependencyError public for tests in another package

* Create tests for errors in eval results

* Extract logic to pull frame errors out into exported function

* Maybe we can drop cyclomatic complexity lint suppression now?

* extract frame errors and fail recording rules if frames contain error

* Fix up retry logic to actually work

* Do not retry non retryable errors

pkg/services/ngalert/eval/eval.go

@@ -141,9 +141,6 @@ type ExecutionResults struct {
 	// Results contains the results of all queries, reduce and math expressions
 	Results map[string]data.Frames
 
-	// Errors contains a map of RefIDs that returned an error
-	Errors map[string]error
-
 	// NoData contains the DatasourceUID for RefIDs that returned no data.
 	NoData map[string]string
 
@@ -166,16 +163,12 @@ func (evalResults Results) HasErrors() bool {
 // HasNonRetryableErrors returns true if we have at least 1 result with:
 // 1. A `State` of `Error`
 // 2. The `Error` attribute is not nil
-// 3. The `Error` type is of `&invalidEvalResultFormatError` or `ErrSeriesMustBeWide`
+// 3. The `Error` matches IsNonRetryableError
 // Our thinking with this approach, is that we don't want to retry errors that have relation with invalid alert definition format.
 func (evalResults Results) HasNonRetryableErrors() bool {
 	for _, r := range evalResults {
 		if r.State == Error && r.Error != nil {
-			var nonRetryableError *invalidEvalResultFormatError
-			if errors.As(r.Error, &nonRetryableError) {
-				return true
-			}
-			if errors.Is(r.Error, expr.ErrSeriesMustBeWide) {
+			if IsNonRetryableError(r.Error) {
 				return true
 			}
 		}
@@ -183,6 +176,19 @@ func (evalResults Results) HasNonRetryableErrors() bool {
 	return false
 }
 
+// IsNonRetryableError indicates whether an error is considered persistent and not worth performing evaluation retries.
+// Currently it is true if err is `&invalidEvalResultFormatError` or `ErrSeriesMustBeWide`
+func IsNonRetryableError(err error) bool {
+	var nonRetryableError *invalidEvalResultFormatError
+	if errors.As(err, &nonRetryableError) {
+		return true
+	}
+	if errors.Is(err, expr.ErrSeriesMustBeWide) {
+		return true
+	}
+	return false
+}
+
 // HasErrors returns true when Results contains at least one element and all elements are errors
 func (evalResults Results) IsError() bool {
 	for _, r := range evalResults {
@@ -407,7 +413,6 @@ type NumberValueCapture struct {
 	Value *float64
 }
 
-//nolint:gocyclo
 func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.QueryDataResponse) ExecutionResults {
 	// captures contains the values of all instant queries and expressions for each dimension
 	captures := make(map[string]map[data.Fingerprint]NumberValueCapture)
@@ -433,17 +438,10 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 	}
 
 	result := ExecutionResults{Results: make(map[string]data.Frames)}
-	for refID, res := range execResp.Responses {
-		if res.Error != nil {
-			if result.Errors == nil {
-				result.Errors = make(map[string]error)
-			}
-			result.Errors[refID] = res.Error
-			if refID == c.Condition {
-				result.Error = res.Error
-			}
-		}
 
+	result.Error = FindConditionError(execResp, c.Condition)
+
+	for refID, res := range execResp.Responses {
 		// There are two possible frame formats for No Data:
 		//
 		// 1. A response with no frames
@@ -526,30 +524,50 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 		}
 	}
 
+	return result
+}
+
+// FindConditionError extracts the error from a query response that caused the given condition to fail.
+// If a condition failed because a node it depends on had an error, that error is returned instead.
+// It returns nil if there are no errors related to the condition.
+func FindConditionError(resp *backend.QueryDataResponse, condition string) error {
+	if resp == nil {
+		return nil
+	}
+
+	errs := make(map[string]error)
+	for refID, node := range resp.Responses {
+		if node.Error != nil {
+			errs[refID] = node.Error
+		}
+	}
+
+	conditionErr := errs[condition]
+
 	// If the error of the condition is an Error that indicates the condition failed
 	// because one of its dependent query or expressions failed, then we follow
 	// the dependency chain to an error that is not a dependency error.
-	if len(result.Errors) > 0 && result.Error != nil {
-		if errors.Is(result.Error, expr.DependencyError) {
+	if conditionErr != nil {
+		if errors.Is(conditionErr, expr.DependencyError) {
 			var utilError errutil.Error
-			e := result.Error
+			e := conditionErr
 			for {
 				errors.As(e, &utilError)
 				depRefID := utilError.PublicPayload["depRefId"].(string)
-				depError, ok := result.Errors[depRefID]
+				depError, ok := errs[depRefID]
 				if !ok {
-					return result
+					return conditionErr
 				}
 				if !errors.Is(depError, expr.DependencyError) {
-					result.Error = depError
-					return result
+					conditionErr = depError
+					return conditionErr
 				}
 				e = depError
 			}
 		}
 	}
 
-	return result
+	return conditionErr
 }
 
 // datasourceUIDsToRefIDs returns a sorted slice of Ref IDs for each Datasource UID.