Alerting: Recording rules understands errors embedded in dataframes (#88946)

* Make MakeDependencyError public for tests in another package

* Create tests for errors in eval results

* Extract logic to pull frame errors out into exported function

* Maybe we can drop cyclomatic complexity lint suppression now?

* extract frame errors and fail recording rules if frames contain error

* Fix up retry logic to actually work

* Do not retry non retryable errors

pkg/services/ngalert/eval/eval_test.go

@@ -894,6 +894,85 @@ func TestEvaluate(t *testing.T) {
 			},
 			EvaluationString: "[ var='A' labels={foo=bar} value=10 ], [ var='B' labels={bar=baz, foo=bar} value=1 ]",
 		}},
+	}, {
+		name: "results contains error if condition frame has error",
+		cond: models.Condition{
+			Condition: "B",
+		},
+		resp: backend.QueryDataResponse{
+			Responses: backend.Responses{
+				"A": {
+					Frames: []*data.Frame{{
+						RefID: "A",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar"},
+								[]*float64{util.Pointer(10.0)},
+							),
+						},
+					}},
+				},
+				"B": {
+					Frames: []*data.Frame{{
+						RefID: "B",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar", "bar": "baz"},
+								[]*float64{util.Pointer(1.0)},
+							),
+						},
+					}},
+					Error: errors.New("some frame error"),
+				},
+			},
+		},
+		expected: Results{{
+			State:            Error,
+			Error:            errors.New("some frame error"),
+			EvaluationString: "",
+		}},
+	}, {
+		name: "results contain underlying error if condition frame has error that depends on another node",
+		cond: models.Condition{
+			Condition: "B",
+		},
+		resp: backend.QueryDataResponse{
+			Responses: backend.Responses{
+				"A": {
+					Frames: []*data.Frame{{
+						RefID: "A",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar"},
+								[]*float64{util.Pointer(10.0)},
+							),
+						},
+					}},
+					Error: errors.New("another error depends on me"),
+				},
+				"B": {
+					Frames: []*data.Frame{{
+						RefID: "B",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar", "bar": "baz"},
+								[]*float64{util.Pointer(1.0)},
+							),
+						},
+					}},
+					Error: expr.MakeDependencyError("B", "A"),
+				},
+			},
+		},
+		expected: Results{{
+			State:            Error,
+			Error:            errors.New("another error depends on me"),
+			EvaluationString: "",
+		}},
 	}}
 
 	for _, tc := range cases {