Alerting: Recording rules understands errors embedded in dataframes (#88946)

* Make MakeDependencyError public for tests in another package * Create tests for errors in eval results * Extract logic to pull frame errors out into exported function * Maybe we can drop cyclomatic complexity lint suppression now? * extract frame errors and fail recording rules if frames contain error * Fix up retry logic to actually work * Do not retry non retryable errors
2024-06-11 10:37:10 -05:00
parent 45c601acc4
commit d004f8a98d
5 changed files with 165 additions and 54 deletions
@@ -141,9 +141,6 @@ type ExecutionResults struct {
 	// Results contains the results of all queries, reduce and math expressions
 	Results map[string]data.Frames

-	// Errors contains a map of RefIDs that returned an error
-	Errors map[string]error
-
 	// NoData contains the DatasourceUID for RefIDs that returned no data.
 	NoData map[string]string

@@ -166,16 +163,12 @@ func (evalResults Results) HasErrors() bool {
 // HasNonRetryableErrors returns true if we have at least 1 result with:
 // 1. A `State` of `Error`
 // 2. The `Error` attribute is not nil
-// 3. The `Error` type is of `&invalidEvalResultFormatError` or `ErrSeriesMustBeWide`
+// 3. The `Error` matches IsNonRetryableError
 // Our thinking with this approach, is that we don't want to retry errors that have relation with invalid alert definition format.
 func (evalResults Results) HasNonRetryableErrors() bool {
 	for _, r := range evalResults {
 		if r.State == Error && r.Error != nil {
-			var nonRetryableError *invalidEvalResultFormatError
-			if errors.As(r.Error, &nonRetryableError) {
-				return true
-			}
-			if errors.Is(r.Error, expr.ErrSeriesMustBeWide) {
+			if IsNonRetryableError(r.Error) {
 				return true
 			}
 		}
@@ -183,6 +176,19 @@ func (evalResults Results) HasNonRetryableErrors() bool {
 	return false
 }

+// IsNonRetryableError indicates whether an error is considered persistent and not worth performing evaluation retries.
+// Currently it is true if err is `&invalidEvalResultFormatError` or `ErrSeriesMustBeWide`
+func IsNonRetryableError(err error) bool {
+	var nonRetryableError *invalidEvalResultFormatError
+	if errors.As(err, &nonRetryableError) {
+		return true
+	}
+	if errors.Is(err, expr.ErrSeriesMustBeWide) {
+		return true
+	}
+	return false
+}
+
 // HasErrors returns true when Results contains at least one element and all elements are errors
 func (evalResults Results) IsError() bool {
 	for _, r := range evalResults {
@@ -407,7 +413,6 @@ type NumberValueCapture struct {
 	Value *float64
 }

-//nolint:gocyclo
 func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.QueryDataResponse) ExecutionResults {
 	// captures contains the values of all instant queries and expressions for each dimension
 	captures := make(map[string]map[data.Fingerprint]NumberValueCapture)
@@ -433,17 +438,10 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 	}

 	result := ExecutionResults{Results: make(map[string]data.Frames)}
-	for refID, res := range execResp.Responses {
-		if res.Error != nil {
-			if result.Errors == nil {
-				result.Errors = make(map[string]error)
-			}
-			result.Errors[refID] = res.Error
-			if refID == c.Condition {
-				result.Error = res.Error
-			}
-		}

+	result.Error = FindConditionError(execResp, c.Condition)
+
+	for refID, res := range execResp.Responses {
 		// There are two possible frame formats for No Data:
 		//
 		// 1. A response with no frames
@@ -526,30 +524,50 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 		}
 	}

+	return result
+}
+
+// FindConditionError extracts the error from a query response that caused the given condition to fail.
+// If a condition failed because a node it depends on had an error, that error is returned instead.
+// It returns nil if there are no errors related to the condition.
+func FindConditionError(resp *backend.QueryDataResponse, condition string) error {
+	if resp == nil {
+		return nil
+	}
+
+	errs := make(map[string]error)
+	for refID, node := range resp.Responses {
+		if node.Error != nil {
+			errs[refID] = node.Error
+		}
+	}
+
+	conditionErr := errs[condition]
+
 	// If the error of the condition is an Error that indicates the condition failed
 	// because one of its dependent query or expressions failed, then we follow
 	// the dependency chain to an error that is not a dependency error.
-	if len(result.Errors) > 0 && result.Error != nil {
-		if errors.Is(result.Error, expr.DependencyError) {
+	if conditionErr != nil {
+		if errors.Is(conditionErr, expr.DependencyError) {
 			var utilError errutil.Error
-			e := result.Error
+			e := conditionErr
 			for {
 				errors.As(e, &utilError)
 				depRefID := utilError.PublicPayload["depRefId"].(string)
-				depError, ok := result.Errors[depRefID]
+				depError, ok := errs[depRefID]
 				if !ok {
-					return result
+					return conditionErr
 				}
 				if !errors.Is(depError, expr.DependencyError) {
-					result.Error = depError
-					return result
+					conditionErr = depError
+					return conditionErr
 				}
 				e = depError
 			}
 		}
 	}

-	return result
+	return conditionErr
 }

 // datasourceUIDsToRefIDs returns a sorted slice of Ref IDs for each Datasource UID.
@@ -894,6 +894,85 @@ func TestEvaluate(t *testing.T) {
 			},
 			EvaluationString: "[ var='A' labels={foo=bar} value=10 ], [ var='B' labels={bar=baz, foo=bar} value=1 ]",
 		}},
+	}, {
+		name: "results contains error if condition frame has error",
+		cond: models.Condition{
+			Condition: "B",
+		},
+		resp: backend.QueryDataResponse{
+			Responses: backend.Responses{
+				"A": {
+					Frames: []*data.Frame{{
+						RefID: "A",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar"},
+								[]*float64{util.Pointer(10.0)},
+							),
+						},
+					}},
+				},
+				"B": {
+					Frames: []*data.Frame{{
+						RefID: "B",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar", "bar": "baz"},
+								[]*float64{util.Pointer(1.0)},
+							),
+						},
+					}},
+					Error: errors.New("some frame error"),
+				},
+			},
+		},
+		expected: Results{{
+			State:            Error,
+			Error:            errors.New("some frame error"),
+			EvaluationString: "",
+		}},
+	}, {
+		name: "results contain underlying error if condition frame has error that depends on another node",
+		cond: models.Condition{
+			Condition: "B",
+		},
+		resp: backend.QueryDataResponse{
+			Responses: backend.Responses{
+				"A": {
+					Frames: []*data.Frame{{
+						RefID: "A",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar"},
+								[]*float64{util.Pointer(10.0)},
+							),
+						},
+					}},
+					Error: errors.New("another error depends on me"),
+				},
+				"B": {
+					Frames: []*data.Frame{{
+						RefID: "B",
+						Fields: []*data.Field{
+							data.NewField(
+								"Value",
+								data.Labels{"foo": "bar", "bar": "baz"},
+								[]*float64{util.Pointer(1.0)},
+							),
+						},
+					}},
+					Error: expr.MakeDependencyError("B", "A"),
+				},
+			},
+		},
+		expected: Results{{
+			State:            Error,
+			Error:            errors.New("another error depends on me"),
+			EvaluationString: "",
+		}},
 	}}

 	for _, tc := range cases {