public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: rule backtesting API (#57318)

Browse Source 
* Implement backtesting engine that can process regular rule specification (with queries to datasource) as well as special kind of rules that have data frame instead of query.
* declare a new API endpoint and model
* add feature toggle `alertingBacktesting`
This commit is contained in:
Yuri Tseretyan
2022-12-14 09:44:14 -05:00
committed by GitHub
parent 258696409d
commit ad09feed83
24 changed files with 1871 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api.go
+8
View File
@@ -10,7 +10,9 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasourceproxy"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/backtesting"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -81,6 +83,9 @@ type API struct {
AlertRules *provisioning.AlertRuleService
AlertsRouter *sender.AlertsRouter
EvaluatorFactory eval.EvaluatorFactory
FeatureManager featuremgmt.FeatureToggles
AppUrl *url.URL
}
// RegisterAPIEndpoints registers API handlers
@@ -126,6 +131,9 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
log: logger,
accessControl: api.AccessControl,
evaluator: api.EvaluatorFactory,
cfg: &api.Cfg.UnifiedAlerting,
backtesting: backtesting.NewEngine(api.AppUrl, api.EvaluatorFactory),
featureManager: api.FeatureManager,
}), m)
api.RegisterConfigurationApiEndpoints(NewConfiguration(
&ConfigSrv{
pkg/services/ngalert/api/api_testing.go
+76
View File
@@ -1,10 +1,12 @@
package api
import (
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
@@ -13,9 +15,12 @@ import (
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/backtesting"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
@@ -25,6 +30,9 @@ type TestingApiSrv struct {
log log.Logger
accessControl accesscontrol.AccessControl
evaluator eval.EvaluatorFactory
cfg *setting.UnifiedAlertingSettings
backtesting *backtesting.Engine
featureManager featuremgmt.FeatureToggles
}
func (srv TestingApiSrv) RouteTestGrafanaRuleConfig(c *models.ReqContext, body apimodels.TestRulePayload) response.Response {
@@ -136,3 +144,71 @@ func (srv TestingApiSrv) RouteEvalQueries(c *models.ReqContext, cmd apimodels.Ev
return response.JSONStreaming(http.StatusOK, evalResults)
}
func (srv TestingApiSrv) BacktestAlertRule(c *models.ReqContext, cmd apimodels.BacktestConfig) response.Response {
if !srv.featureManager.IsEnabled(featuremgmt.FlagAlertingBacktesting) {
return ErrResp(http.StatusNotFound, nil, "Backgtesting API is not enabled")
}
if cmd.From.After(cmd.To) {
return ErrResp(400, nil, "From cannot be greater than To")
}
noDataState, err := ngmodels.NoDataStateFromString(string(cmd.NoDataState))
if err != nil {
return ErrResp(400, err, "")
}
forInterval := time.Duration(cmd.For)
if forInterval < 0 {
return ErrResp(400, nil, "Bad For interval")
}
intervalSeconds, err := validateInterval(srv.cfg, time.Duration(cmd.Interval))
if err != nil {
return ErrResp(400, err, "")
}
if !authorizeDatasourceAccessForRule(&ngmodels.AlertRule{Data: cmd.Data}, func(evaluator accesscontrol.Evaluator) bool {
return accesscontrol.HasAccess(srv.accessControl, c)(accesscontrol.ReqSignedIn, evaluator)
}) {
return errorToResponse(fmt.Errorf("%w to query one or many data sources used by the rule", ErrAuthorization))
}
rule := &ngmodels.AlertRule{
// ID: 0,
// Updated: time.Time{},
// Version: 0,
// NamespaceUID: "",
// DashboardUID: nil,
// PanelID: nil,
// RuleGroup: "",
// RuleGroupIndex: 0,
// ExecErrState: "",
Title: cmd.Title,
// prefix backtesting- is to distinguish between executions of regular rule and backtesting in logs (like expression engine, evaluator, state manager etc)
UID: "backtesting-" + util.GenerateShortUID(),
OrgID: c.OrgID,
Condition: cmd.Condition,
Data: cmd.Data,
IntervalSeconds: intervalSeconds,
NoDataState: noDataState,
For: forInterval,
Annotations: cmd.Annotations,
Labels: cmd.Labels,
}
result, err := srv.backtesting.Test(c.Req.Context(), c.SignedInUser, rule, cmd.From, cmd.To)
if err != nil {
if errors.Is(err, backtesting.ErrInvalidInputData) {
return ErrResp(400, err, "Failed to evaluate")
}
return ErrResp(500, err, "Failed to evaluate")
}
body, err := data.FrameToJSON(result, data.IncludeAll)
if err != nil {
return ErrResp(500, err, "Failed to convert frame to JSON")
}
return response.JSON(http.StatusOK, body)
}
pkg/services/ngalert/api/authorization.go
+5
View File
@@ -68,6 +68,11 @@ func (api *API) authorize(method, path string) web.Handler {
fallback = middleware.ReqSignedIn
// additional authorization is done in the request handler
eval = ac.EvalPermission(ac.ActionAlertingRuleRead)
// Grafana Rules Testing Paths
case http.MethodPost + "/api/v1/rule/backtest":
fallback = middleware.ReqSignedIn
// additional authorization is done in the request handler
eval = ac.EvalPermission(ac.ActionAlertingRuleRead)
case http.MethodPost + "/api/v1/eval":
fallback = middleware.ReqSignedIn
// additional authorization is done in the request handler
pkg/services/ngalert/api/authorization_test.go
+1 -1
View File
@@ -49,7 +49,7 @@ func TestAuthorize(t *testing.T) {
}
paths[p] = methods
}
require.Len(t, paths, 40)
require.Len(t, paths, 41)
ac := acmock.New()
api := &API{AccessControl: ac}
pkg/services/ngalert/api/generated_base_api_testing.go
+19
View File
@@ -19,11 +19,20 @@ import (
)
type TestingApi interface {
BacktestConfig(*models.ReqContext) response.Response
RouteEvalQueries(*models.ReqContext) response.Response
RouteTestRuleConfig(*models.ReqContext) response.Response
RouteTestRuleGrafanaConfig(*models.ReqContext) response.Response
}
func (f *TestingApiHandler) BacktestConfig(ctx *models.ReqContext) response.Response {
// Parse Request Body
conf := apimodels.BacktestConfig{}
if err := web.Bind(ctx.Req, &conf); err != nil {
return response.Error(http.StatusBadRequest, "bad request data", err)
}
return f.handleBacktestingConfig(ctx, conf)
}
func (f *TestingApiHandler) RouteEvalQueries(ctx *models.ReqContext) response.Response {
// Parse Request Body
conf := apimodels.EvalQueriesPayload{}
@@ -53,6 +62,16 @@ func (f *TestingApiHandler) RouteTestRuleGrafanaConfig(ctx *models.ReqContext) r
func (api *API) RegisterTestingApiEndpoints(srv TestingApi, m *metrics.API) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Post(
toMacaronPath("/api/v1/rule/backtest"),
api.authorize(http.MethodPost, "/api/v1/rule/backtest"),
metrics.Instrument(
http.MethodPost,
"/api/v1/rule/backtest",
srv.BacktestConfig,
m,
),
)
group.Post(
toMacaronPath("/api/v1/eval"),
api.authorize(http.MethodPost, "/api/v1/eval"),
pkg/services/ngalert/api/testing_api.go
+4
View File
@@ -28,3 +28,7 @@ func (f *TestingApiHandler) handleRouteTestRuleGrafanaConfig(c *models.ReqContex
func (f *TestingApiHandler) handleRouteEvalQueries(c *models.ReqContext, body apimodels.EvalQueriesPayload) response.Response {
return f.svc.RouteEvalQueries(c, body)
}
func (f *TestingApiHandler) handleBacktestingConfig(ctx *models.ReqContext, conf apimodels.BacktestConfig) response.Response {
return f.svc.BacktestAlertRule(ctx, conf)
}
pkg/services/ngalert/api/tooling/api.json
+54
View File
@@ -289,6 +289,60 @@
"title": "Authorization contains HTTP authorization credentials.",
"type": "object"
},
"BacktestConfig": {
"properties": {
"annotations": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"condition": {
"type": "string"
},
"data": {
"items": {
"$ref": "#/definitions/AlertQuery"
},
"type": "array"
},
"for": {
"$ref": "#/definitions/Duration"
},
"from": {
"format": "date-time",
"type": "string"
},
"interval": {
"$ref": "#/definitions/Duration"
},
"labels": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"no_data_state": {
"enum": [
"Alerting",
"NoData",
"OK"
],
"type": "string"
},
"title": {
"type": "string"
},
"to": {
"format": "date-time",
"type": "string"
}
},
"type": "object"
},
"BacktestResult": {
"$ref": "#/definitions/Frame"
},
"BasicAuth": {
"properties": {
"password": {
pkg/services/ngalert/api/tooling/definitions/testing.go
+35
View File
@@ -6,6 +6,8 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/common/model"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/prometheus/promql"
@@ -53,6 +55,19 @@ import (
// Responses:
// 200: EvalQueriesResponse
// swagger:route Post /api/v1/rule/backtest testing BacktestConfig
//
// Test rule
//
// Consumes:
// - application/json
//
// Produces:
// - application/json
//
// Responses:
// 200: BacktestResult
// swagger:parameters RouteTestReceiverConfig
type TestReceiverRequest struct {
// in:body
@@ -160,3 +175,23 @@ type Failure ResponseDetails
type ResponseDetails struct {
Msg string `json:"msg"`
}
// swagger:model
type BacktestConfig struct {
From time.Time `json:"from"`
To time.Time `json:"to"`
Interval model.Duration `json:"interval,omitempty"`
Condition string `json:"condition"`
Data []models.AlertQuery `json:"data"` // TODO yuri. Create API model for AlertQuery
For model.Duration `json:"for,omitempty"`
Title string `json:"title"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
NoDataState NoDataState `json:"no_data_state"`
}
// swagger:model
type BacktestResult data.Frame
pkg/services/ngalert/api/tooling/post.json
+77
View File
@@ -289,6 +289,60 @@
"title": "Authorization contains HTTP authorization credentials.",
"type": "object"
},
"BacktestConfig": {
"properties": {
"annotations": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"condition": {
"type": "string"
},
"data": {
"items": {
"$ref": "#/definitions/AlertQuery"
},
"type": "array"
},
"for": {
"$ref": "#/definitions/Duration"
},
"from": {
"format": "date-time",
"type": "string"
},
"interval": {
"$ref": "#/definitions/Duration"
},
"labels": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"no_data_state": {
"enum": [
"Alerting",
"NoData",
"OK"
],
"type": "string"
},
"title": {
"type": "string"
},
"to": {
"format": "date-time",
"type": "string"
}
},
"type": "object"
},
"BacktestResult": {
"$ref": "#/definitions/Frame"
},
"BasicAuth": {
"properties": {
"password": {
@@ -6162,6 +6216,29 @@
]
}
},
"/api/v1/rule/backtest": {
"post": {
"consumes": [
"application/json"
],
"description": "Test rule",
"operationId": "BacktestConfig",
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "BacktestResult",
"schema": {
"$ref": "#/definitions/BacktestResult"
}
}
},
"tags": [
"testing"
]
}
},
"/api/v1/rule/test/grafana": {
"post": {
"consumes": [
pkg/services/ngalert/api/tooling/spec.json
+77
View File
@@ -2384,6 +2384,29 @@
}
}
},
"/api/v1/rule/backtest": {
"post": {
"description": "Test rule",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"testing"
],
"operationId": "BacktestConfig",
"responses": {
"200": {
"description": "BacktestResult",
"schema": {
"$ref": "#/definitions/BacktestResult"
}
}
}
}
},
"/api/v1/rule/test/grafana": {
"post": {
"description": "Test a rule against Grafana ruler",
@@ -2748,6 +2771,60 @@
}
}
},
"BacktestConfig": {
"type": "object",
"properties": {
"annotations": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"condition": {
"type": "string"
},
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/AlertQuery"
}
},
"for": {
"$ref": "#/definitions/Duration"
},
"from": {
"type": "string",
"format": "date-time"
},
"interval": {
"$ref": "#/definitions/Duration"
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"no_data_state": {
"type": "string",
"enum": [
"Alerting",
"NoData",
"OK"
]
},
"title": {
"type": "string"
},
"to": {
"type": "string",
"format": "date-time"
}
}
},
"BacktestResult": {
"$ref": "#/definitions/Frame"
},
"BasicAuth": {
"type": "object",
"title": "BasicAuth contains basic HTTP authentication credentials.",