public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Rule backtesting with experimental UI (#115525)

Browse Source 
* add function to convert StateTransition to LokiEntry
* add QueryResultBuilder
* update backtesting to produce result similar to historian
* make shouldRecord public
* filter out noop transitions
* add experimental front-end
* add new fields
* move conversion of api model to AlertRule to validation
* add extra labels
* calculate tick timestamp using the same logic as in scheduler
* implement correct logic of calculating first evaluation timestamp
* add uid, group and folder uid they are needed for jitter strategy

* add JitterOffsetInDuration and JitterStrategy.String()

* add config `backtesting_max_evaluations` to [unified_alerting] (not documented for now)

* remove obsolete tests

* elevate permisisons for backtesting endpoint
* move backtesting to separate dir
This commit is contained in:
Yuri Tseretyan
2025-12-26 16:55:57 -05:00
committed by GitHub
parent e38f007d30
commit fa1e6cce5e
25 changed files with 964 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api.go
+1 -1
View File
@@ -161,7 +161,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
authz: ruleAuthzService,
evaluator: api.EvaluatorFactory,
cfg: &api.Cfg.UnifiedAlerting,
backtesting: backtesting.NewEngine(api.AppUrl, api.EvaluatorFactory, api.Tracer),
backtesting: backtesting.NewEngine(api.AppUrl, api.EvaluatorFactory, api.Tracer, api.Cfg.UnifiedAlerting, api.FeatureManager),
featureManager: api.FeatureManager,
appUrl: api.AppUrl,
tracer: api.Tracer,
pkg/services/ngalert/api/api_testing.go
+14 -46
View File
@@ -34,7 +34,6 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
type folderService interface {
@@ -230,54 +229,27 @@ func (srv TestingApiSrv) BacktestAlertRule(c *contextmodel.ReqContext, cmd apimo
return ErrResp(http.StatusNotFound, nil, "Backgtesting API is not enabled")
}
if cmd.From.After(cmd.To) {
return ErrResp(400, nil, "From cannot be greater than To")
}
noDataState, err := ngmodels.NoDataStateFromString(string(cmd.NoDataState))
rule, err := apivalidation.ValidateBacktestConfig(c.GetOrgID(), cmd, apivalidation.RuleLimitsFromConfig(srv.cfg, srv.featureManager))
if err != nil {
return ErrResp(400, err, "")
}
forInterval := time.Duration(cmd.For)
if forInterval < 0 {
return ErrResp(400, nil, "Bad For interval")
return ErrResp(http.StatusBadRequest, err, "")
}
intervalSeconds, err := apivalidation.ValidateInterval(time.Duration(cmd.Interval), srv.cfg.BaseInterval)
if err != nil {
return ErrResp(400, err, "")
}
queries := AlertQueriesFromApiAlertQueries(cmd.Data)
if err := srv.authz.AuthorizeDatasourceAccessForRule(c.Req.Context(), c.SignedInUser, &ngmodels.AlertRule{Data: queries}); err != nil {
if err := srv.authz.AuthorizeDatasourceAccessForRule(c.Req.Context(), c.SignedInUser, rule); err != nil {
return errorToResponse(err)
}
rule := &ngmodels.AlertRule{
// ID: 0,
// Updated: time.Time{},
// Version: 0,
// NamespaceUID: "",
// DashboardUID: nil,
// PanelID: nil,
// RuleGroup: "",
// RuleGroupIndex: 0,
// ExecErrState: "",
Title: cmd.Title,
// prefix backtesting- is to distinguish between executions of regular rule and backtesting in logs (like expression engine, evaluator, state manager etc)
UID: "backtesting-" + util.GenerateShortUID(),
OrgID: c.GetOrgID(),
Condition: cmd.Condition,
Data: queries,
IntervalSeconds: intervalSeconds,
NoDataState: noDataState,
For: forInterval,
Annotations: cmd.Annotations,
Labels: cmd.Labels,
// Fetch folder path for alert labels, fallback to "Backtesting" if not available
var folderTitle string
if cmd.NamespaceUID != "" {
f, err := srv.folderService.GetNamespaceByUID(c.Req.Context(), cmd.NamespaceUID, c.OrgID, c.SignedInUser)
if err != nil {
srv.log.FromContext(c.Req.Context()).Warn("Failed to fetch folder path for alert labels", "error", err)
} else {
folderTitle = f.Fullpath
}
}
result, err := srv.backtesting.Test(c.Req.Context(), c.SignedInUser, rule, cmd.From, cmd.To)
result, err := srv.backtesting.Test(c.Req.Context(), c.SignedInUser, rule, cmd.From, cmd.To, folderTitle)
if err != nil {
if errors.Is(err, backtesting.ErrInvalidInputData) {
return ErrResp(400, err, "Failed to evaluate")
@@ -285,9 +257,5 @@ func (srv TestingApiSrv) BacktestAlertRule(c *contextmodel.ReqContext, cmd apimo
return ErrResp(500, err, "Failed to evaluate")
}
body, err := data.FrameToJSON(result, data.IncludeAll)
if err != nil {
return ErrResp(500, err, "Failed to convert frame to JSON")
}
return response.JSON(http.StatusOK, body)
return response.JSONStreaming(http.StatusOK, result)
}
pkg/services/ngalert/api/authorization.go
+8 -2
View File
@@ -81,9 +81,15 @@ func (api *API) authorize(method, path string) web.Handler {
// additional authorization is done in the request handler
eval = ac.EvalPermission(ac.ActionAlertingRuleRead)
// Grafana Rules Testing Paths
case http.MethodPost + "/api/v1/rule/backtest":
case http.MethodPost + "/api/v1/rule/backtest": // TODO (yuri) this should be protected by dedicated permission
// additional authorization is done in the request handler
eval = ac.EvalPermission(ac.ActionAlertingRuleRead)
eval = ac.EvalAll(
ac.EvalPermission(ac.ActionAlertingRuleRead),
ac.EvalAny(
ac.EvalPermission(ac.ActionAlertingRuleUpdate),
ac.EvalPermission(ac.ActionAlertingRuleCreate),
),
)
case http.MethodPost + "/api/v1/eval":
// additional authorization is done in the request handler
eval = ac.EvalPermission(ac.ActionAlertingRuleRead)
pkg/services/ngalert/api/tooling/definitions/testing.go
+13 -7
View File
@@ -221,15 +221,21 @@ type BacktestConfig struct {
To time.Time `json:"to"`
Interval model.Duration `json:"interval,omitempty"`
Condition string `json:"condition"`
Data []AlertQuery `json:"data"`
For model.Duration `json:"for,omitempty"`
Condition string `json:"condition"`
Data []AlertQuery `json:"data"`
For *model.Duration `json:"for,omitempty"`
KeepFiringFor *model.Duration `json:"keep_firing_for,omitempty"`
Title string `json:"title"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Title string `json:"title"`
Labels map[string]string `json:"labels,omitempty"`
NoDataState NoDataState `json:"no_data_state"`
NoDataState NoDataState `json:"no_data_state"`
ExecErrState ExecutionErrorState `json:"exec_err_state"`
MissingSeriesEvalsToResolve *int64 `json:"missing_series_evals_to_resolve,omitempty"`
UID string `json:"uid,omitempty"`
RuleGroup string `json:"rule_group,omitempty"`
NamespaceUID string `json:"namespace_uid,omitempty"`
}
// swagger:model
pkg/services/ngalert/api/validation/api_ruler_validation.go
+47 -10
View File
@@ -249,6 +249,21 @@ func ValidateCondition(condition string, queries []apimodels.AlertQuery, canPatc
return nil
}
func validateGroupInterval(incoming prommodels.Duration, limits RuleLimits) (time.Duration, error) {
interval := time.Duration(incoming)
if interval == 0 {
// if group interval is 0 (undefined) then we automatically fall back to the default interval
interval = limits.DefaultRuleEvaluationInterval
}
if interval < 0 || int64(interval.Seconds())%int64(limits.BaseInterval.Seconds()) != 0 {
return 0, fmt.Errorf("rule evaluation interval (%d second) should be positive number that is multiple of the base interval of %d seconds", int64(interval.Seconds()), int64(limits.BaseInterval.Seconds()))
}
// TODO should we validate that interval is >= cfg.MinInterval? Currently, we allow to save but fix the specified interval if it is < cfg.MinInterval
return interval, nil
}
func ValidateInterval(interval, baseInterval time.Duration) (int64, error) {
intervalSeconds := int64(interval.Seconds())
@@ -336,18 +351,11 @@ func ValidateRuleGroup(
return nil, fmt.Errorf("rule group name is too long. Max length is %d", store.AlertRuleMaxRuleGroupNameLength)
}
interval := time.Duration(ruleGroupConfig.Interval)
if interval == 0 {
// if group interval is 0 (undefined) then we automatically fall back to the default interval
interval = limits.DefaultRuleEvaluationInterval
interval, err := validateGroupInterval(ruleGroupConfig.Interval, limits)
if err != nil {
return nil, err
}
if interval < 0 || int64(interval.Seconds())%int64(limits.BaseInterval.Seconds()) != 0 {
return nil, fmt.Errorf("rule evaluation interval (%d second) should be positive number that is multiple of the base interval of %d seconds", int64(interval.Seconds()), int64(limits.BaseInterval.Seconds()))
}
// TODO should we validate that interval is >= cfg.MinInterval? Currently, we allow to save but fix the specified interval if it is < cfg.MinInterval
// If the rule group is reserved for no-group rules, we cannot have multiple rules in it.
if isNoGroupRuleGroup && len(ruleGroupConfig.Rules) > 1 {
return nil, fmt.Errorf("rule group %s is reserved for no-group rules and cannot be used for rule groups with multiple rules", ruleGroupConfig.Name)
@@ -410,3 +418,32 @@ func ValidateNotificationSettings(n *apimodels.AlertRuleNotificationSettings) ([
s,
}, nil
}
func ValidateBacktestConfig(orgId int64, config apimodels.BacktestConfig, limits RuleLimits) (*ngmodels.AlertRule, error) {
if config.From.After(config.To) {
return nil, fmt.Errorf("invalid testing range: from %s must be before to %s", config.From, config.To)
}
interval, err := validateGroupInterval(config.Interval, limits)
if err != nil {
return nil, err
}
return ValidateRuleNode(&apimodels.PostableExtendedRuleNode{
ApiRuleNode: &apimodels.ApiRuleNode{
For: config.For,
KeepFiringFor: config.KeepFiringFor,
Labels: config.Labels,
Annotations: nil,
},
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
Title: config.Title,
Condition: config.Condition,
Data: config.Data,
UID: config.UID,
NoDataState: config.NoDataState,
ExecErrState: config.ExecErrState,
MissingSeriesEvalsToResolve: config.MissingSeriesEvalsToResolve,
},
}, config.RuleGroup, interval, orgId, config.NamespaceUID, limits)
}