public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Rule backtesting with experimental UI (#115525)

Browse Source 
* add function to convert StateTransition to LokiEntry
* add QueryResultBuilder
* update backtesting to produce result similar to historian
* make shouldRecord public
* filter out noop transitions
* add experimental front-end
* add new fields
* move conversion of api model to AlertRule to validation
* add extra labels
* calculate tick timestamp using the same logic as in scheduler
* implement correct logic of calculating first evaluation timestamp
* add uid, group and folder uid they are needed for jitter strategy

* add JitterOffsetInDuration and JitterStrategy.String()

* add config `backtesting_max_evaluations` to [unified_alerting] (not documented for now)

* remove obsolete tests

* elevate permisisons for backtesting endpoint
* move backtesting to separate dir
This commit is contained in:
Yuri Tseretyan
2025-12-26 16:55:57 -05:00
committed by GitHub
parent e38f007d30
commit fa1e6cce5e
25 changed files with 964 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/backtesting/engine.go
+181 -54
View File
@@ -15,10 +15,16 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/state/historian"
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
var (
@@ -28,7 +34,7 @@ var (
backtestingEvaluatorFactory = newBacktestingEvaluator
)
type callbackFunc = func(evaluationIndex int, now time.Time, results eval.Results) error
type callbackFunc = func(evaluationIndex int, now time.Time, results eval.Results) (bool, error)
type backtestingEvaluator interface {
Eval(ctx context.Context, from time.Time, interval time.Duration, evaluations int, callback callbackFunc) error
@@ -40,11 +46,17 @@ type stateManager interface {
}
type Engine struct {
evalFactory eval.EvaluatorFactory
createStateManager func() stateManager
evalFactory eval.EvaluatorFactory
createStateManager func() stateManager
disableGrafanaFolder bool
featureToggles featuremgmt.FeatureToggles
minInterval time.Duration
baseInterval time.Duration
jitterStrategy schedule.JitterStrategy
maxEvaluations int
}
func NewEngine(appUrl *url.URL, evalFactory eval.EvaluatorFactory, tracer tracing.Tracer) *Engine {
func NewEngine(appUrl *url.URL, evalFactory eval.EvaluatorFactory, tracer tracing.Tracer, cfg setting.UnifiedAlertingSettings, toggles featuremgmt.FeatureToggles) *Engine {
return &Engine{
evalFactory: evalFactory,
createStateManager: func() stateManager {
@@ -60,74 +72,139 @@ func NewEngine(appUrl *url.URL, evalFactory eval.EvaluatorFactory, tracer tracin
}
return state.NewManager(cfg, state.NewNoopPersister())
},
disableGrafanaFolder: false,
featureToggles: toggles,
minInterval: cfg.MinInterval,
baseInterval: cfg.BaseInterval,
maxEvaluations: cfg.BacktestingMaxEvaluations,
jitterStrategy: schedule.JitterStrategyFrom(cfg, toggles),
}
}
func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models.AlertRule, from, to time.Time) (*data.Frame, error) {
ruleCtx := models.WithRuleKey(ctx, rule.GetKey())
logger := logger.FromContext(ctx)
func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models.AlertRule, from, to time.Time, folderTitle string) (res *data.Frame, err error) {
if rule == nil {
return nil, fmt.Errorf("%w: rule is not defined", ErrInvalidInputData)
}
if !from.Before(to) {
return nil, fmt.Errorf("%w: invalid interval of the backtesting [%d,%d]", ErrInvalidInputData, from.Unix(), to.Unix())
return nil, fmt.Errorf("%w: invalid interval [%d,%d]", ErrInvalidInputData, from.Unix(), to.Unix())
}
if to.Sub(from).Seconds() < float64(rule.IntervalSeconds) {
return nil, fmt.Errorf("%w: interval of the backtesting [%d,%d] is less than evaluation interval [%ds]", ErrInvalidInputData, from.Unix(), to.Unix(), rule.IntervalSeconds)
ruleCtx := models.WithRuleKey(ctx, rule.GetKey())
logger := logger.FromContext(ruleCtx).New("backtesting", util.GenerateShortUID())
var warns []string
if rule.GetInterval() < e.minInterval {
logger.Warn("Interval adjusted to minimal interval", "originalInterval", rule.GetInterval(), "adjustedInterval", e.minInterval)
rule = rule.Copy()
rule.IntervalSeconds = int64(e.minInterval.Seconds())
warns = append(warns, fmt.Sprintf("Interval adjusted to minimal interval %ds", rule.IntervalSeconds))
}
length := int(to.Sub(from).Seconds()) / int(rule.IntervalSeconds)
stateManager := e.createStateManager()
effectiveStrategy := e.jitterStrategy
if e.jitterStrategy == schedule.JitterByGroup && (rule.RuleGroup == "" || rule.NamespaceUID == "") ||
e.jitterStrategy == schedule.JitterByRule && rule.UID == "" {
logger.Warn(fmt.Sprintf("Jitter strategy is set to %s, but rule group or namespace is not set. Ignore jitter", e.jitterStrategy))
warns = append(warns, fmt.Sprintf("Jitter strategy is set to %s, but rule group or namespace is not set. Ignore jitter. The results of testing will be different than real evaluations", e.jitterStrategy))
effectiveStrategy = schedule.JitterNever
}
jitterOffset := schedule.JitterOffsetInDuration(rule, e.baseInterval, effectiveStrategy)
firstEval, err := getFirstEvaluationTime(from, rule, e.baseInterval, jitterOffset)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrInvalidInputData, err)
}
evaluator, err := backtestingEvaluatorFactory(ruleCtx, e.evalFactory, user, rule.GetEvalCondition().WithSource("backtesting"), &schedule.AlertingResultsFromRuleState{
Manager: stateManager,
Rule: rule,
})
evaluations := calculateNumberOfEvaluations(firstEval, to, rule.GetInterval())
if e.maxEvaluations > 0 && evaluations > e.maxEvaluations {
logger.Warn("Evaluations adjusted to maximal number", "originalEvaluations", evaluations, "adjustedEvaluations", e.maxEvaluations)
warns = append(warns, fmt.Sprintf("Number of evaluations are adjusted to the limit of %d evaluations. Requested: %d", e.maxEvaluations, evaluations))
evaluations = e.maxEvaluations
}
start := time.Now()
defer func() {
if err == nil {
logger.Info("Rule testing finished successfully", "duration", time.Since(start))
} else {
logger.Error("Rule testing finished with error", "duration", time.Since(start), "error", err)
}
}()
stateMgr := e.createStateManager()
evaluator, err := backtestingEvaluatorFactory(ruleCtx,
e.evalFactory,
user,
rule.GetEvalCondition().WithSource("backtesting"),
&schedule.AlertingResultsFromRuleState{
Manager: stateMgr,
Rule: rule,
},
)
if err != nil {
return nil, errors.Join(ErrInvalidInputData, err)
}
logger.Info("Start testing alert rule", "from", from, "to", to, "interval", rule.IntervalSeconds, "evaluations", length)
logger.Info("Start testing alert rule", "from", from, "to", to, "interval", rule.GetInterval(), "firstTick", firstEval, "evaluations", evaluations, "jitterOffset", jitterOffset, "jitterStrategy", effectiveStrategy)
start := time.Now()
var builder *historian.QueryResultBuilder
tsField := data.NewField("Time", nil, make([]time.Time, length))
valueFields := make(map[data.Fingerprint]*data.Field)
err = evaluator.Eval(ruleCtx, from, time.Duration(rule.IntervalSeconds)*time.Second, length, func(idx int, currentTime time.Time, results eval.Results) error {
if idx >= length {
logger.Info("Unexpected evaluation. Skipping", "from", from, "to", to, "interval", rule.IntervalSeconds, "evaluationTime", currentTime, "evaluationIndex", idx, "expectedEvaluations", length)
return nil
}
states := stateManager.ProcessEvalResults(ruleCtx, currentTime, rule, results, nil, nil)
tsField.Set(idx, currentTime)
for _, s := range states {
field, ok := valueFields[s.CacheID]
if !ok {
field = data.NewField("", s.Labels, make([]*string, length))
valueFields[s.CacheID] = field
}
if s.State.State != eval.NoData { // set nil if NoData
value := s.State.State.String()
if s.StateReason != "" {
value += " (" + s.StateReason + ")"
}
field.Set(idx, &value)
continue
}
}
return nil
})
fields := make([]*data.Field, 0, len(valueFields)+1)
fields = append(fields, tsField)
for _, f := range valueFields {
fields = append(fields, f)
ruleMeta := history_model.RuleMeta{
ID: rule.ID,
OrgID: rule.OrgID,
UID: rule.UID,
Title: rule.Title,
Group: rule.RuleGroup,
NamespaceUID: rule.NamespaceUID,
// DashboardUID: "",
// PanelID: 0,
Condition: rule.Condition,
}
result := data.NewFrame("Testing results", fields...)
labels := map[string]string{
historian.OrgIDLabel: fmt.Sprint(ruleMeta.OrgID),
historian.GroupLabel: fmt.Sprint(ruleMeta.Group),
historian.FolderUIDLabel: fmt.Sprint(rule.NamespaceUID),
}
labelsBytes, err := json.Marshal(labels)
if err != nil {
return nil, err
}
logger.Info("Rule testing finished successfully", "duration", time.Since(start))
return result, nil
// Ensure fallback if empty string is passed
if folderTitle == "" {
folderTitle = "Backtesting"
}
extraLabels := state.GetRuleExtraLabels(logger, rule, folderTitle, !e.disableGrafanaFolder, e.featureToggles)
processFn := func(idx int, currentTime time.Time, results eval.Results) (bool, error) {
// init the builder. Do the best guess for the size of the result
if builder == nil {
builder = historian.NewQueryResultBuilder(evaluations * len(results))
for _, warn := range warns {
builder.AddWarn(warn)
}
}
states := stateMgr.ProcessEvalResults(ruleCtx, currentTime, rule, results, extraLabels, nil)
for _, s := range states {
if !historian.ShouldRecord(s) {
continue
}
entry := historian.StateTransitionToLokiEntry(ruleMeta, s)
err := builder.AddRow(currentTime, entry, labelsBytes)
if err != nil {
return false, err
}
}
return idx <= evaluations, nil
}
err = evaluator.Eval(ruleCtx, firstEval, rule.GetInterval(), evaluations, processFn)
if err != nil {
return nil, err
}
if builder == nil {
return nil, errors.New("no results were produced")
}
return builder.ToFrame(), nil
}
func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition, reader eval.AlertingResultsReader) (backtestingEvaluator, error) {
@@ -173,3 +250,53 @@ type NoopImageService struct{}
func (s *NoopImageService) NewImage(_ context.Context, _ *models.AlertRule) (*models.Image, error) {
return &models.Image{}, nil
}
func getNextEvaluationTime(currentTime time.Time, rule *models.AlertRule, baseInterval time.Duration, jitterOffset time.Duration) (time.Time, error) {
if rule.IntervalSeconds%int64(baseInterval.Seconds()) != 0 {
return time.Time{}, fmt.Errorf("interval %ds is not divisible by base interval %ds", rule.IntervalSeconds, int64(baseInterval.Seconds()))
}
freq := rule.IntervalSeconds / int64(baseInterval.Seconds())
firstTickNum := currentTime.Unix() / int64(baseInterval.Seconds())
jitterOffsetTicks := int64(jitterOffset / baseInterval)
firstEvalTickNum := firstTickNum + (jitterOffsetTicks-(firstTickNum%freq)+freq)%freq
return time.Unix(firstEvalTickNum*int64(baseInterval.Seconds()), 0), nil
}
func getFirstEvaluationTime(from time.Time, rule *models.AlertRule, baseInterval time.Duration, jitterOffset time.Duration) (time.Time, error) {
// Now calculate the time of the tick the same way as in the scheduler
firstTick := ticker.GetStartTick(from, baseInterval)
// calculate time of the first evaluation that is at or after the first tick
firstEval, err := getNextEvaluationTime(firstTick, rule, baseInterval, jitterOffset)
if err != nil {
return time.Time{}, err
}
// Ensure firstEval is at or after from
// Calculate how many intervals to skip to get past 'from'
if firstEval.Before(from) {
diff := from.Sub(firstEval)
interval := rule.GetInterval()
// Ceiling division: how many intervals needed to cover the difference
intervalsToAdd := (diff + interval - 1) / interval
firstEval = firstEval.Add(interval * intervalsToAdd)
}
return firstEval, nil
}
func calculateNumberOfEvaluations(firstEval, to time.Time, interval time.Duration) int {
var evaluations int
if to.After(firstEval) {
evaluations = int(to.Sub(firstEval).Seconds()) / int(interval.Seconds())
}
if evaluations == 0 {
evaluations = 1
}
return evaluations
}
pkg/services/ngalert/backtesting/engine_test.go
+191 -147
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand"
"testing"
"time"
@@ -14,9 +13,11 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/eval/eval_mocks"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/util"
)
@@ -158,16 +159,6 @@ func TestNewBacktestingEvaluator(t *testing.T) {
}
func TestEvaluatorTest(t *testing.T) {
states := []eval.State{eval.Normal, eval.Alerting, eval.Pending}
generateState := func(prefix string) *state.State {
labels := models.GenerateAlertLabels(rand.Intn(5)+1, prefix+"-")
return &state.State{
CacheID: labels.Fingerprint(),
Labels: labels,
State: states[rand.Intn(len(states))],
}
}
randomResultCallback := func(now time.Time) (eval.Results, error) {
return eval.GenerateResults(rand.Intn(5)+1, eval.ResultGen()), nil
}
@@ -189,84 +180,17 @@ func TestEvaluatorTest(t *testing.T) {
createStateManager: func() stateManager {
return manager
},
disableGrafanaFolder: false,
featureToggles: featuremgmt.WithFeatures(),
minInterval: 1 * time.Second,
baseInterval: 1 * time.Second,
jitterStrategy: schedule.JitterNever,
maxEvaluations: 10000,
}
gen := models.RuleGen
rule := gen.With(gen.WithInterval(time.Second)).GenerateRef()
ruleInterval := time.Duration(rule.IntervalSeconds) * time.Second
t.Run("should return data frame in specific format", func(t *testing.T) {
from := time.Unix(0, 0)
to := from.Add(5 * ruleInterval)
allStates := [...]eval.State{eval.Normal, eval.Alerting, eval.Pending, eval.NoData, eval.Error}
var states []state.StateTransition
for _, s := range allStates {
labels := models.GenerateAlertLabels(rand.Intn(5)+1, s.String()+"-")
states = append(states, state.StateTransition{
State: &state.State{
CacheID: labels.Fingerprint(),
Labels: labels,
State: s,
StateReason: util.GenerateShortUID(),
},
})
}
manager.stateCallback = func(now time.Time) []state.StateTransition {
return states
}
frame, err := engine.Test(context.Background(), nil, rule, from, to)
require.NoError(t, err)
require.Len(t, frame.Fields, len(states)+1) // +1 - timestamp
t.Run("should contain field Time", func(t *testing.T) {
timestampField, _ := frame.FieldByName("Time")
require.NotNil(t, timestampField, "frame does not contain field 'Time'")
require.Equal(t, data.FieldTypeTime, timestampField.Type())
})
fieldByState := make(map[data.Fingerprint]*data.Field, len(states))
t.Run("should contain a field per state", func(t *testing.T) {
for _, s := range states {
var f *data.Field
for _, field := range frame.Fields {
if field.Labels.String() == s.Labels.String() {
f = field
break
}
}
require.NotNilf(t, f, "Cannot find a field by state labels")
fieldByState[s.CacheID] = f
}
})
t.Run("should be populated with correct values", func(t *testing.T) {
timestampField, _ := frame.FieldByName("Time")
expectedLength := timestampField.Len()
for _, field := range frame.Fields {
require.Equalf(t, expectedLength, field.Len(), "Field %s should have the size %d", field.Name, expectedLength)
}
for i := 0; i < expectedLength; i++ {
expectedTime := from.Add(time.Duration(int64(i)*rule.IntervalSeconds) * time.Second)
require.Equal(t, expectedTime, timestampField.At(i).(time.Time))
for _, s := range states {
f := fieldByState[s.CacheID]
if s.State.State == eval.NoData {
require.Nil(t, f.At(i))
} else {
v := f.At(i).(*string)
require.NotNilf(t, v, "Field [%s] value at index %d should not be nil", s.CacheID, i)
require.Equal(t, fmt.Sprintf("%s (%s)", s.State.State, s.StateReason), *v)
}
}
}
})
})
t.Run("should not fail if 'to-from' is not times of interval", func(t *testing.T) {
from := time.Unix(0, 0)
to := from.Add(5 * ruleInterval)
@@ -287,84 +211,26 @@ func TestEvaluatorTest(t *testing.T) {
return states
}
frame, err := engine.Test(context.Background(), nil, rule, from, to)
frame, err := engine.Test(context.Background(), nil, rule, from, to, "")
require.NoError(t, err)
expectedLen := frame.Rows()
for i := 0; i < 100; i++ {
jitter := time.Duration(rand.Int63n(ruleInterval.Milliseconds())) * time.Millisecond
frame, err = engine.Test(context.Background(), nil, rule, from, to.Add(jitter))
frame, err = engine.Test(context.Background(), nil, rule, from, to.Add(jitter), "")
require.NoError(t, err)
require.Equalf(t, expectedLen, frame.Rows(), "jitter %v caused result to be different that base-line", jitter)
}
})
t.Run("should backfill field with nulls if a new dimension created in the middle", func(t *testing.T) {
from := time.Unix(0, 0)
state1 := state.StateTransition{
State: generateState("1"),
}
state2 := state.StateTransition{
State: generateState("2"),
}
state3 := state.StateTransition{
State: generateState("3"),
}
stateByTime := map[time.Time][]state.StateTransition{
from: {state1, state2},
from.Add(1 * ruleInterval): {state1, state2},
from.Add(2 * ruleInterval): {state1, state2},
from.Add(3 * ruleInterval): {state1, state2, state3},
from.Add(4 * ruleInterval): {state1, state2, state3},
}
to := from.Add(time.Duration(len(stateByTime)) * ruleInterval)
manager.stateCallback = func(now time.Time) []state.StateTransition {
return stateByTime[now]
}
frame, err := engine.Test(context.Background(), nil, rule, from, to)
require.NoError(t, err)
var field3 *data.Field
for _, field := range frame.Fields {
if field.Labels.String() == state3.Labels.String() {
field3 = field
break
}
}
require.NotNilf(t, field3, "Result for state 3 was not found")
require.Equalf(t, len(stateByTime), field3.Len(), "State3 result has unexpected number of values")
idx := 0
for curTime, states := range stateByTime {
value := field3.At(idx).(*string)
if len(states) == 2 {
require.Nilf(t, value, "The result should be nil if state3 was not available for time %v", curTime)
}
}
})
t.Run("should fail", func(t *testing.T) {
manager.stateCallback = func(now time.Time) []state.StateTransition {
return nil
}
t.Run("when interval is not correct", func(t *testing.T) {
from := time.Now()
t.Run("when from=to", func(t *testing.T) {
to := from
_, err := engine.Test(context.Background(), nil, rule, from, to)
require.ErrorIs(t, err, ErrInvalidInputData)
})
t.Run("when from > to", func(t *testing.T) {
to := from.Add(-ruleInterval)
_, err := engine.Test(context.Background(), nil, rule, from, to)
require.ErrorIs(t, err, ErrInvalidInputData)
})
t.Run("when to-from < interval", func(t *testing.T) {
to := from.Add(ruleInterval).Add(-time.Millisecond)
_, err := engine.Test(context.Background(), nil, rule, from, to)
_, err := engine.Test(context.Background(), nil, rule, from, to, "")
require.ErrorIs(t, err, ErrInvalidInputData)
})
})
@@ -376,7 +242,7 @@ func TestEvaluatorTest(t *testing.T) {
}
from := time.Now()
to := from.Add(ruleInterval)
_, err := engine.Test(context.Background(), nil, rule, from, to)
_, err := engine.Test(context.Background(), nil, rule, from, to, "")
require.ErrorIs(t, err, expectedError)
})
})
@@ -404,10 +270,188 @@ func (f *fakeBacktestingEvaluator) Eval(_ context.Context, from time.Time, inter
if err != nil {
return err
}
err = callback(idx, now, results)
c, err := callback(idx, now, results)
if err != nil {
return err
}
if !c {
break
}
}
return nil
}
func TestGetNextEvaluationTime(t *testing.T) {
baseInterval := 10 * time.Second
testCases := []struct {
name string
ruleInterval int64
currentTimestamp int64
jitterOffset time.Duration
expectError bool
expectedNext int64
}{
{
name: "interval not divisible by base interval",
ruleInterval: 15,
currentTimestamp: 0,
jitterOffset: 0,
expectError: true,
},
{
name: "no jitter - from tick 0",
ruleInterval: 20,
currentTimestamp: 0,
jitterOffset: 0,
expectedNext: 0,
},
{
name: "no jitter - from tick 1",
ruleInterval: 20,
currentTimestamp: 10,
jitterOffset: 0,
expectedNext: 20,
},
{
name: "no jitter - from tick 2",
ruleInterval: 20,
currentTimestamp: 20,
jitterOffset: 0,
expectedNext: 20,
},
{
name: "with 20s jitter - from tick 0",
ruleInterval: 60,
currentTimestamp: 0,
jitterOffset: 20 * time.Second,
expectedNext: 20,
},
{
name: "with 20s jitter - from tick 2",
ruleInterval: 60,
currentTimestamp: 20,
jitterOffset: 20 * time.Second,
expectedNext: 20,
},
{
name: "with 20s jitter - from tick 3",
ruleInterval: 60,
currentTimestamp: 30,
jitterOffset: 20 * time.Second,
expectedNext: 80,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
rule := &models.AlertRule{IntervalSeconds: tc.ruleInterval}
currentTime := time.Unix(tc.currentTimestamp, 0)
result, err := getNextEvaluationTime(currentTime, rule, baseInterval, tc.jitterOffset)
if tc.expectError {
require.Error(t, err)
require.Contains(t, err.Error(), "is not divisible by base interval")
return
}
require.NoError(t, err)
require.Equal(t, tc.expectedNext, result.Unix())
})
}
}
func TestGetFirstEvaluationTime(t *testing.T) {
baseInterval := 10 * time.Second
testCases := []struct {
name string
ruleInterval int64
fromUnix int64
jitterOffset time.Duration
expectError bool
expectedUnix int64
}{
{
name: "interval not divisible by base interval",
ruleInterval: 15,
fromUnix: 0,
jitterOffset: 0,
expectError: true,
},
{
name: "no jitter - from at tick 0",
ruleInterval: 20,
fromUnix: 0,
jitterOffset: 0,
expectedUnix: 0,
},
{
name: "no jitter - from at tick 1",
ruleInterval: 20,
fromUnix: 10,
jitterOffset: 0,
expectedUnix: 20,
},
{
name: "no jitter - from before first tick",
ruleInterval: 20,
fromUnix: 5,
jitterOffset: 0,
expectedUnix: 20,
},
{
name: "no jitter - from after first aligned tick",
ruleInterval: 20,
fromUnix: 25,
jitterOffset: 0,
expectedUnix: 40,
},
{
name: "no jitter - from at tick boundary",
ruleInterval: 10,
fromUnix: 10,
jitterOffset: 0,
expectedUnix: 10,
},
{
name: "with 20s jitter - from epoch",
ruleInterval: 60,
fromUnix: 0,
jitterOffset: 20 * time.Second,
expectedUnix: 20,
},
{
name: "with 20s jitter - from 70s",
ruleInterval: 60,
fromUnix: 70,
jitterOffset: 20 * time.Second,
expectedUnix: 80,
},
{
name: "with 50s jitter - from 25s",
ruleInterval: 60,
fromUnix: 25,
jitterOffset: 50 * time.Second,
expectedUnix: 50,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
rule := &models.AlertRule{IntervalSeconds: tc.ruleInterval}
from := time.Unix(tc.fromUnix, 0)
result, err := getFirstEvaluationTime(from, rule, baseInterval, tc.jitterOffset)
if tc.expectError {
require.Error(t, err)
require.Contains(t, err.Error(), "is not divisible by base interval")
return
}
require.NoError(t, err)
require.Equal(t, tc.expectedUnix, result.Unix())
require.GreaterOrEqual(t, result.Unix(), from.Unix(), "first eval should be at or after from")
})
}
}
pkg/services/ngalert/backtesting/eval_data.go
+4 -1
View File
@@ -85,10 +85,13 @@ func (d *dataEvaluator) Eval(_ context.Context, from time.Time, interval time.Du
EvaluatedAt: now,
})
}
err := callback(i, now, result)
cont, err := callback(i, now, result)
if err != nil {
return err
}
if !cont {
break
}
}
return nil
}
pkg/services/ngalert/backtesting/eval_data_test.go
+22 -13
View File
@@ -100,11 +100,11 @@ func TestDataEvaluator_Eval(t *testing.T) {
resultsCount := int(to.Sub(from).Seconds() / interval.Seconds())
err = evaluator.Eval(context.Background(), from, time.Second, resultsCount, func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), from, time.Second, resultsCount, func(idx int, now time.Time, res eval.Results) (bool, error) {
r = append(r, results{
now, res,
})
return nil
return true, nil
})
require.NoError(t, err)
@@ -164,11 +164,11 @@ func TestDataEvaluator_Eval(t *testing.T) {
size := to.Sub(from).Milliseconds() / interval.Milliseconds()
r := make([]results, 0, size)
err = evaluator.Eval(context.Background(), from, interval, int(size), func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), from, interval, int(size), func(idx int, now time.Time, res eval.Results) (bool, error) {
r = append(r, results{
now, res,
})
return nil
return true, nil
})
currentRowIdx := 0
@@ -195,11 +195,11 @@ func TestDataEvaluator_Eval(t *testing.T) {
size := int(to.Sub(from).Seconds() / interval.Seconds())
r := make([]results, 0, size)
err = evaluator.Eval(context.Background(), from, interval, size, func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), from, interval, size, func(idx int, now time.Time, res eval.Results) (bool, error) {
r = append(r, results{
now, res,
})
return nil
return true, nil
})
currentRowIdx := 0
@@ -230,11 +230,11 @@ func TestDataEvaluator_Eval(t *testing.T) {
t.Run("should be noData until the frame interval", func(t *testing.T) {
newFrom := from.Add(-10 * time.Second)
r := make([]results, 0, int(to.Sub(newFrom).Seconds()))
err = evaluator.Eval(context.Background(), newFrom, time.Second, cap(r), func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), newFrom, time.Second, cap(r), func(idx int, now time.Time, res eval.Results) (bool, error) {
r = append(r, results{
now, res,
})
return nil
return true, nil
})
rowIdx := 0
@@ -258,11 +258,11 @@ func TestDataEvaluator_Eval(t *testing.T) {
t.Run("should be the last value after the frame interval", func(t *testing.T) {
newTo := to.Add(10 * time.Second)
r := make([]results, 0, int(newTo.Sub(from).Seconds()))
err = evaluator.Eval(context.Background(), from, time.Second, cap(r), func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), from, time.Second, cap(r), func(idx int, now time.Time, res eval.Results) (bool, error) {
r = append(r, results{
now, res,
})
return nil
return true, nil
})
rowIdx := 0
@@ -282,12 +282,21 @@ func TestDataEvaluator_Eval(t *testing.T) {
})
t.Run("should stop if callback error", func(t *testing.T) {
expectedError := errors.New("error")
err = evaluator.Eval(context.Background(), from, time.Second, 6, func(idx int, now time.Time, res eval.Results) error {
err = evaluator.Eval(context.Background(), from, time.Second, 6, func(idx int, now time.Time, res eval.Results) (bool, error) {
if idx == 5 {
return expectedError
return false, expectedError
}
return nil
return true, nil
})
require.ErrorIs(t, err, expectedError)
})
t.Run("should stop if callback does not want to continue", func(t *testing.T) {
evaluated := 0
err = evaluator.Eval(context.Background(), from, time.Second, 6, func(idx int, now time.Time, res eval.Results) (bool, error) {
evaluated++
return evaluated < 2, nil
})
require.NoError(t, err)
require.Equal(t, 2, evaluated)
})
}
pkg/services/ngalert/backtesting/eval_query.go
+4 -1
View File
@@ -18,10 +18,13 @@ func (d *queryEvaluator) Eval(ctx context.Context, from time.Time, interval time
if err != nil {
return err
}
err = callback(idx, now, results)
cont, err := callback(idx, now, results)
if err != nil {
return err
}
if !cont {
break
}
}
return nil
}
pkg/services/ngalert/backtesting/eval_query_test.go
+25 -8
View File
@@ -31,9 +31,9 @@ func TestQueryEvaluator_Eval(t *testing.T) {
intervals := make([]time.Time, times)
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) error {
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) (bool, error) {
intervals[idx] = now
return nil
return true, nil
})
require.NoError(t, err)
require.Len(t, intervals, times)
@@ -49,7 +49,7 @@ func TestQueryEvaluator_Eval(t *testing.T) {
}
})
t.Run("should stop evaluation if error", func(t *testing.T) {
t.Run("should stop evaluation", func(t *testing.T) {
t.Run("when evaluation fails", func(t *testing.T) {
m := &eval_mocks.ConditionEvaluatorMock{}
expectedResults := eval.Results{}
@@ -62,9 +62,9 @@ func TestQueryEvaluator_Eval(t *testing.T) {
intervals := make([]time.Time, 0, times)
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) error {
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) (bool, error) {
intervals = append(intervals, now)
return nil
return true, nil
})
require.ErrorIs(t, err, expectedError)
require.Len(t, intervals, 3)
@@ -81,14 +81,31 @@ func TestQueryEvaluator_Eval(t *testing.T) {
intervals := make([]time.Time, 0, times)
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) error {
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) (bool, error) {
if len(intervals) > 3 {
return expectedError
return false, expectedError
}
intervals = append(intervals, now)
return nil
return true, nil
})
require.ErrorIs(t, err, expectedError)
})
t.Run("when callback does not want to continue", func(t *testing.T) {
m := &eval_mocks.ConditionEvaluatorMock{}
expectedResults := eval.Results{}
m.EXPECT().Evaluate(mock.Anything, mock.Anything).Return(expectedResults, nil)
evaluator := queryEvaluator{
eval: m,
}
evaluated := 0
err := evaluator.Eval(ctx, from, interval, times, func(idx int, now time.Time, results eval.Results) (bool, error) {
evaluated++
return evaluated <= 2, nil
})
require.NoError(t, err, nil)
require.Equal(t, 3, evaluated)
})
})
}