Alerting: Do not retry rule evaluations with "input data must be a wide series but got type long" style errors (#87343)

add typed error for series must be wide, do not retry

pkg/services/ngalert/eval/eval.go

@@ -166,7 +166,7 @@ func (evalResults Results) HasErrors() bool {
 // HasNonRetryableErrors returns true if we have at least 1 result with:
 // 1. A `State` of `Error`
 // 2. The `Error` attribute is not nil
-// 3. The `Error` type is of `&invalidEvalResultFormatError`
+// 3. The `Error` type is of `&invalidEvalResultFormatError` or `ErrSeriesMustBeWide`
 // Our thinking with this approach, is that we don't want to retry errors that have relation with invalid alert definition format.
 func (evalResults Results) HasNonRetryableErrors() bool {
 	for _, r := range evalResults {
@@ -175,6 +175,9 @@ func (evalResults Results) HasNonRetryableErrors() bool {
 			if errors.As(r.Error, &nonRetryableError) {
 				return true
 			}
+			if errors.Is(r.Error, expr.ErrSeriesMustBeWide) {
+				return true
+			}
 		}
 	}
 	return false

pkg/services/ngalert/eval/eval_test.go

@@ -957,7 +957,7 @@ func TestResults_HasNonRetryableErrors(t *testing.T) {
 		expected bool
 	}{
 		{
-			name: "with non-retryable errors",
+			name: "with invalid format error",
 			eval: Results{
 				{
 					State: Error,
@@ -966,6 +966,16 @@ func TestResults_HasNonRetryableErrors(t *testing.T) {
 			},
 			expected: true,
 		},
+		{
+			name: "with expected wide series but got type long error",
+			eval: Results{
+				{
+					State: Error,
+					Error: fmt.Errorf("%w but got type long", expr.ErrSeriesMustBeWide),
+				},
+			},
+			expected: true,
+		},
 		{
 			name: "with retryable errors",
 			eval: Results{