Alerting: Rule backtesting with experimental UI (#115525)

* add function to convert StateTransition to LokiEntry
* add QueryResultBuilder
* update backtesting to produce result similar to historian
* make shouldRecord public
* filter out noop transitions
* add experimental front-end
* add new fields
* move conversion of api model to AlertRule to validation
* add extra labels
* calculate tick timestamp using the same logic as in scheduler
* implement correct logic of calculating first evaluation timestamp
* add uid, group and folder uid they are needed for jitter strategy

* add JitterOffsetInDuration and JitterStrategy.String()

* add config `backtesting_max_evaluations` to [unified_alerting] (not documented for now)

* remove obsolete tests

* elevate permisisons for backtesting endpoint
* move backtesting to separate dir

pkg/services/ngalert/backtesting/engine_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"math/rand"
 	"testing"
 	"time"
@@ -14,9 +13,11 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval/eval_mocks"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -158,16 +159,6 @@ func TestNewBacktestingEvaluator(t *testing.T) {
 }
 
 func TestEvaluatorTest(t *testing.T) {
-	states := []eval.State{eval.Normal, eval.Alerting, eval.Pending}
-	generateState := func(prefix string) *state.State {
-		labels := models.GenerateAlertLabels(rand.Intn(5)+1, prefix+"-")
-		return &state.State{
-			CacheID: labels.Fingerprint(),
-			Labels:  labels,
-			State:   states[rand.Intn(len(states))],
-		}
-	}
-
 	randomResultCallback := func(now time.Time) (eval.Results, error) {
 		return eval.GenerateResults(rand.Intn(5)+1, eval.ResultGen()), nil
 	}
@@ -189,84 +180,17 @@ func TestEvaluatorTest(t *testing.T) {
 		createStateManager: func() stateManager {
 			return manager
 		},
+		disableGrafanaFolder: false,
+		featureToggles:       featuremgmt.WithFeatures(),
+		minInterval:          1 * time.Second,
+		baseInterval:         1 * time.Second,
+		jitterStrategy:       schedule.JitterNever,
+		maxEvaluations:       10000,
 	}
 	gen := models.RuleGen
 	rule := gen.With(gen.WithInterval(time.Second)).GenerateRef()
 	ruleInterval := time.Duration(rule.IntervalSeconds) * time.Second
 
-	t.Run("should return data frame in specific format", func(t *testing.T) {
-		from := time.Unix(0, 0)
-		to := from.Add(5 * ruleInterval)
-		allStates := [...]eval.State{eval.Normal, eval.Alerting, eval.Pending, eval.NoData, eval.Error}
-
-		var states []state.StateTransition
-
-		for _, s := range allStates {
-			labels := models.GenerateAlertLabels(rand.Intn(5)+1, s.String()+"-")
-			states = append(states, state.StateTransition{
-				State: &state.State{
-					CacheID:     labels.Fingerprint(),
-					Labels:      labels,
-					State:       s,
-					StateReason: util.GenerateShortUID(),
-				},
-			})
-		}
-
-		manager.stateCallback = func(now time.Time) []state.StateTransition {
-			return states
-		}
-
-		frame, err := engine.Test(context.Background(), nil, rule, from, to)
-
-		require.NoError(t, err)
-		require.Len(t, frame.Fields, len(states)+1) // +1 - timestamp
-
-		t.Run("should contain field Time", func(t *testing.T) {
-			timestampField, _ := frame.FieldByName("Time")
-			require.NotNil(t, timestampField, "frame does not contain field 'Time'")
-			require.Equal(t, data.FieldTypeTime, timestampField.Type())
-		})
-
-		fieldByState := make(map[data.Fingerprint]*data.Field, len(states))
-
-		t.Run("should contain a field per state", func(t *testing.T) {
-			for _, s := range states {
-				var f *data.Field
-				for _, field := range frame.Fields {
-					if field.Labels.String() == s.Labels.String() {
-						f = field
-						break
-					}
-				}
-				require.NotNilf(t, f, "Cannot find a field by state labels")
-				fieldByState[s.CacheID] = f
-			}
-		})
-
-		t.Run("should be populated with correct values", func(t *testing.T) {
-			timestampField, _ := frame.FieldByName("Time")
-			expectedLength := timestampField.Len()
-			for _, field := range frame.Fields {
-				require.Equalf(t, expectedLength, field.Len(), "Field %s should have the size %d", field.Name, expectedLength)
-			}
-			for i := 0; i < expectedLength; i++ {
-				expectedTime := from.Add(time.Duration(int64(i)*rule.IntervalSeconds) * time.Second)
-				require.Equal(t, expectedTime, timestampField.At(i).(time.Time))
-				for _, s := range states {
-					f := fieldByState[s.CacheID]
-					if s.State.State == eval.NoData {
-						require.Nil(t, f.At(i))
-					} else {
-						v := f.At(i).(*string)
-						require.NotNilf(t, v, "Field [%s] value at index %d should not be nil", s.CacheID, i)
-						require.Equal(t, fmt.Sprintf("%s (%s)", s.State.State, s.StateReason), *v)
-					}
-				}
-			}
-		})
-	})
-
 	t.Run("should not fail if 'to-from' is not times of interval", func(t *testing.T) {
 		from := time.Unix(0, 0)
 		to := from.Add(5 * ruleInterval)
@@ -287,84 +211,26 @@ func TestEvaluatorTest(t *testing.T) {
 			return states
 		}
 
-		frame, err := engine.Test(context.Background(), nil, rule, from, to)
+		frame, err := engine.Test(context.Background(), nil, rule, from, to, "")
 		require.NoError(t, err)
 		expectedLen := frame.Rows()
 		for i := 0; i < 100; i++ {
 			jitter := time.Duration(rand.Int63n(ruleInterval.Milliseconds())) * time.Millisecond
-			frame, err = engine.Test(context.Background(), nil, rule, from, to.Add(jitter))
+			frame, err = engine.Test(context.Background(), nil, rule, from, to.Add(jitter), "")
 			require.NoError(t, err)
 			require.Equalf(t, expectedLen, frame.Rows(), "jitter %v caused result to be different that base-line", jitter)
 		}
 	})
 
-	t.Run("should backfill field with nulls if a new dimension created in the middle", func(t *testing.T) {
-		from := time.Unix(0, 0)
-
-		state1 := state.StateTransition{
-			State: generateState("1"),
-		}
-		state2 := state.StateTransition{
-			State: generateState("2"),
-		}
-		state3 := state.StateTransition{
-			State: generateState("3"),
-		}
-		stateByTime := map[time.Time][]state.StateTransition{
-			from:                       {state1, state2},
-			from.Add(1 * ruleInterval): {state1, state2},
-			from.Add(2 * ruleInterval): {state1, state2},
-			from.Add(3 * ruleInterval): {state1, state2, state3},
-			from.Add(4 * ruleInterval): {state1, state2, state3},
-		}
-		to := from.Add(time.Duration(len(stateByTime)) * ruleInterval)
-
-		manager.stateCallback = func(now time.Time) []state.StateTransition {
-			return stateByTime[now]
-		}
-
-		frame, err := engine.Test(context.Background(), nil, rule, from, to)
-		require.NoError(t, err)
-
-		var field3 *data.Field
-		for _, field := range frame.Fields {
-			if field.Labels.String() == state3.Labels.String() {
-				field3 = field
-				break
-			}
-		}
-		require.NotNilf(t, field3, "Result for state 3 was not found")
-		require.Equalf(t, len(stateByTime), field3.Len(), "State3 result has unexpected number of values")
-
-		idx := 0
-		for curTime, states := range stateByTime {
-			value := field3.At(idx).(*string)
-			if len(states) == 2 {
-				require.Nilf(t, value, "The result should be nil if state3 was not available for time %v", curTime)
-			}
-		}
-	})
-
 	t.Run("should fail", func(t *testing.T) {
 		manager.stateCallback = func(now time.Time) []state.StateTransition {
 			return nil
 		}
-
 		t.Run("when interval is not correct", func(t *testing.T) {
 			from := time.Now()
-			t.Run("when from=to", func(t *testing.T) {
-				to := from
-				_, err := engine.Test(context.Background(), nil, rule, from, to)
-				require.ErrorIs(t, err, ErrInvalidInputData)
-			})
 			t.Run("when from > to", func(t *testing.T) {
 				to := from.Add(-ruleInterval)
-				_, err := engine.Test(context.Background(), nil, rule, from, to)
-				require.ErrorIs(t, err, ErrInvalidInputData)
-			})
-			t.Run("when to-from < interval", func(t *testing.T) {
-				to := from.Add(ruleInterval).Add(-time.Millisecond)
-				_, err := engine.Test(context.Background(), nil, rule, from, to)
+				_, err := engine.Test(context.Background(), nil, rule, from, to, "")
 				require.ErrorIs(t, err, ErrInvalidInputData)
 			})
 		})
@@ -376,7 +242,7 @@ func TestEvaluatorTest(t *testing.T) {
 			}
 			from := time.Now()
 			to := from.Add(ruleInterval)
-			_, err := engine.Test(context.Background(), nil, rule, from, to)
+			_, err := engine.Test(context.Background(), nil, rule, from, to, "")
 			require.ErrorIs(t, err, expectedError)
 		})
 	})
@@ -404,10 +270,188 @@ func (f *fakeBacktestingEvaluator) Eval(_ context.Context, from time.Time, inter
 		if err != nil {
 			return err
 		}
-		err = callback(idx, now, results)
+		c, err := callback(idx, now, results)
 		if err != nil {
 			return err
 		}
+		if !c {
+			break
+		}
 	}
 	return nil
 }
+
+func TestGetNextEvaluationTime(t *testing.T) {
+	baseInterval := 10 * time.Second
+
+	testCases := []struct {
+		name             string
+		ruleInterval     int64
+		currentTimestamp int64
+		jitterOffset     time.Duration
+		expectError      bool
+		expectedNext     int64
+	}{
+		{
+			name:             "interval not divisible by base interval",
+			ruleInterval:     15,
+			currentTimestamp: 0,
+			jitterOffset:     0,
+			expectError:      true,
+		},
+		{
+			name:             "no jitter - from tick 0",
+			ruleInterval:     20,
+			currentTimestamp: 0,
+			jitterOffset:     0,
+			expectedNext:     0,
+		},
+		{
+			name:             "no jitter - from tick 1",
+			ruleInterval:     20,
+			currentTimestamp: 10,
+			jitterOffset:     0,
+			expectedNext:     20,
+		},
+		{
+			name:             "no jitter - from tick 2",
+			ruleInterval:     20,
+			currentTimestamp: 20,
+			jitterOffset:     0,
+			expectedNext:     20,
+		},
+		{
+			name:             "with 20s jitter - from tick 0",
+			ruleInterval:     60,
+			currentTimestamp: 0,
+			jitterOffset:     20 * time.Second,
+			expectedNext:     20,
+		},
+		{
+			name:             "with 20s jitter - from tick 2",
+			ruleInterval:     60,
+			currentTimestamp: 20,
+			jitterOffset:     20 * time.Second,
+			expectedNext:     20,
+		},
+		{
+			name:             "with 20s jitter - from tick 3",
+			ruleInterval:     60,
+			currentTimestamp: 30,
+			jitterOffset:     20 * time.Second,
+			expectedNext:     80,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			rule := &models.AlertRule{IntervalSeconds: tc.ruleInterval}
+			currentTime := time.Unix(tc.currentTimestamp, 0)
+			result, err := getNextEvaluationTime(currentTime, rule, baseInterval, tc.jitterOffset)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "is not divisible by base interval")
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedNext, result.Unix())
+		})
+	}
+}
+
+func TestGetFirstEvaluationTime(t *testing.T) {
+	baseInterval := 10 * time.Second
+
+	testCases := []struct {
+		name         string
+		ruleInterval int64
+		fromUnix     int64
+		jitterOffset time.Duration
+		expectError  bool
+		expectedUnix int64
+	}{
+		{
+			name:         "interval not divisible by base interval",
+			ruleInterval: 15,
+			fromUnix:     0,
+			jitterOffset: 0,
+			expectError:  true,
+		},
+		{
+			name:         "no jitter - from at tick 0",
+			ruleInterval: 20,
+			fromUnix:     0,
+			jitterOffset: 0,
+			expectedUnix: 0,
+		},
+		{
+			name:         "no jitter - from at tick 1",
+			ruleInterval: 20,
+			fromUnix:     10,
+			jitterOffset: 0,
+			expectedUnix: 20,
+		},
+		{
+			name:         "no jitter - from before first tick",
+			ruleInterval: 20,
+			fromUnix:     5,
+			jitterOffset: 0,
+			expectedUnix: 20,
+		},
+		{
+			name:         "no jitter - from after first aligned tick",
+			ruleInterval: 20,
+			fromUnix:     25,
+			jitterOffset: 0,
+			expectedUnix: 40,
+		},
+		{
+			name:         "no jitter - from at tick boundary",
+			ruleInterval: 10,
+			fromUnix:     10,
+			jitterOffset: 0,
+			expectedUnix: 10,
+		},
+		{
+			name:         "with 20s jitter - from epoch",
+			ruleInterval: 60,
+			fromUnix:     0,
+			jitterOffset: 20 * time.Second,
+			expectedUnix: 20,
+		},
+		{
+			name:         "with 20s jitter - from 70s",
+			ruleInterval: 60,
+			fromUnix:     70,
+			jitterOffset: 20 * time.Second,
+			expectedUnix: 80,
+		},
+		{
+			name:         "with 50s jitter - from 25s",
+			ruleInterval: 60,
+			fromUnix:     25,
+			jitterOffset: 50 * time.Second,
+			expectedUnix: 50,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			rule := &models.AlertRule{IntervalSeconds: tc.ruleInterval}
+			from := time.Unix(tc.fromUnix, 0)
+			result, err := getFirstEvaluationTime(from, rule, baseInterval, tc.jitterOffset)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "is not divisible by base interval")
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedUnix, result.Unix())
+			require.GreaterOrEqual(t, result.Unix(), from.Unix(), "first eval should be at or after from")
+		})
+	}
+}