[v9.3.x] Alerting: Fix evaluation timeout (#61307)

Alerting: Fix evaluation timeout (#61303)

(cherry picked from commit b4e1e1871f)

Co-authored-by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>

pkg/services/ngalert/eval/eval.go

@@ -40,9 +40,13 @@ type ConditionEvaluator interface {
 	Evaluate(ctx context.Context, now time.Time) (Results, error)
 }
 
+type expressionService interface {
+	ExecutePipeline(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error)
+}
+
 type conditionEvaluator struct {
 	pipeline          expr.DataPipeline
-	expressionService *expr.Service
+	expressionService expressionService
 	condition         models.Condition
 	evalTimeout       time.Duration
 }
@@ -61,7 +65,7 @@ func (r *conditionEvaluator) EvaluateRaw(ctx context.Context, now time.Time) (re
 	}()
 
 	execCtx := ctx
-	if r.evalTimeout <= 0 {
+	if r.evalTimeout >= 0 {
 		timeoutCtx, cancel := context.WithTimeout(ctx, r.evalTimeout)
 		defer cancel()
 		execCtx = timeoutCtx

pkg/services/ngalert/eval/eval_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/stretchr/testify/require"
 	ptr "github.com/xorcare/pointer"
@@ -455,3 +456,38 @@ func TestValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestEvaluateRaw(t *testing.T) {
+	t.Run("should timeout if request takes too long", func(t *testing.T) {
+		unexpectedResponse := &backend.QueryDataResponse{}
+
+		e := conditionEvaluator{
+			pipeline: nil,
+			expressionService: &fakeExpressionService{
+				hook: func(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error) {
+					ts := time.Now()
+					for time.Since(ts) <= 10*time.Second {
+						if ctx.Err() != nil {
+							return nil, ctx.Err()
+						}
+						time.Sleep(10 * time.Millisecond)
+					}
+					return unexpectedResponse, nil
+				},
+			},
+			condition:   models.Condition{},
+			evalTimeout: 10 * time.Millisecond,
+		}
+
+		_, err := e.EvaluateRaw(context.Background(), time.Now())
+		require.ErrorIs(t, err, context.DeadlineExceeded)
+	})
+}
+
+type fakeExpressionService struct {
+	hook func(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error)
+}
+
+func (f fakeExpressionService) ExecutePipeline(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error) {
+	return f.hook(ctx, now, pipeline)
+}