Alerting: rule backtesting API (#57318)
* Implement backtesting engine that can process regular rule specification (with queries to datasource) as well as special kind of rules that have data frame instead of query. * declare a new API endpoint and model * add feature toggle `alertingBacktesting`
This commit is contained in:
Yuri Tseretyan
@@ -0,0 +1,306 @@
|
||||
{
|
||||
"data": {
|
||||
"from": "2022-10-19T18:44:00Z",
|
||||
"to": "2022-10-19T19:44:00Z",
|
||||
"interval": "1m",
|
||||
"for": "0",
|
||||
"labels": {
|
||||
"templatable-label": "test"
|
||||
},
|
||||
"annotations": {
|
||||
"anno-test": "test"
|
||||
},
|
||||
"condition": "A",
|
||||
"no_data_state": "Alerting",
|
||||
"data": [
|
||||
{
|
||||
"refId": "A",
|
||||
"queryType": "",
|
||||
"datasourceUid": "__data__",
|
||||
"model": {
|
||||
"data": {
|
||||
"schema": {
|
||||
"name": "A-series",
|
||||
"refId": "A",
|
||||
"fields": [
|
||||
{
|
||||
"name": "Time",
|
||||
"type": "time",
|
||||
"typeInfo": {
|
||||
"frame": "time.Time",
|
||||
"nullable": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "A-series",
|
||||
"type": "number",
|
||||
"typeInfo": {
|
||||
"frame": "float64",
|
||||
"nullable": true
|
||||
},
|
||||
"labels": {
|
||||
"label": "2",
|
||||
"test": "1"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"values": [
|
||||
[
|
||||
1666205040000,
|
||||
1666205100000,
|
||||
1666205160000,
|
||||
1666205220000,
|
||||
1666205280000,
|
||||
1666205340000,
|
||||
1666205400000,
|
||||
1666205460000,
|
||||
1666205520000,
|
||||
1666205580000,
|
||||
1666205640000,
|
||||
1666205700000,
|
||||
1666205760000,
|
||||
1666205820000,
|
||||
1666205880000,
|
||||
1666205940000,
|
||||
1666206000000,
|
||||
1666206060000,
|
||||
1666206120000,
|
||||
1666206180000,
|
||||
1666206240000,
|
||||
1666206300000,
|
||||
1666206360000,
|
||||
1666206420000,
|
||||
1666206480000,
|
||||
1666206540000,
|
||||
1666206600000,
|
||||
1666206660000,
|
||||
1666206720000,
|
||||
1666206780000,
|
||||
1666206840000,
|
||||
1666206900000,
|
||||
1666206960000,
|
||||
1666207020000,
|
||||
1666207080000,
|
||||
1666207140000,
|
||||
1666207200000,
|
||||
1666207260000,
|
||||
1666207320000,
|
||||
1666207380000,
|
||||
1666207440000,
|
||||
1666207500000,
|
||||
1666207560000,
|
||||
1666207620000,
|
||||
1666207680000,
|
||||
1666207740000,
|
||||
1666207800000,
|
||||
1666207860000,
|
||||
1666207920000,
|
||||
1666207980000,
|
||||
1666208040000,
|
||||
1666208100000,
|
||||
1666208160000,
|
||||
1666208220000,
|
||||
1666208280000,
|
||||
1666208340000,
|
||||
1666208400000,
|
||||
1666208460000,
|
||||
1666208520000,
|
||||
1666208580000,
|
||||
1666208640000
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"query": {
|
||||
"from": "2022-10-19T18:44:00Z",
|
||||
"to": "2022-10-19T19:44:00Z",
|
||||
"interval": "1m",
|
||||
"for": "5m",
|
||||
"labels": {
|
||||
"templatable-label": "TMPL-{{.Labels.state}}"
|
||||
},
|
||||
"annotations": {
|
||||
"anno-test": "TMPL-ANNO-{{.Labels.state}}"
|
||||
},
|
||||
"condition": "C",
|
||||
"no_data_state": "Alerting",
|
||||
"data": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasourceUid": "testdata",
|
||||
"queryType": "",
|
||||
"relativeTimeRange": {
|
||||
"from": 600,
|
||||
"to": 0
|
||||
},
|
||||
"model": {
|
||||
"refId": "A",
|
||||
"hide": false,
|
||||
"scenarioId": "usa",
|
||||
"usa": {
|
||||
"mode": "timeseries",
|
||||
"period": "1m",
|
||||
"states": [
|
||||
"GA", "FL", "AL", "AZ"
|
||||
],
|
||||
"fields": [
|
||||
"baz"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasourceUid": "-100",
|
||||
"queryType": "",
|
||||
"model": {
|
||||
"refId": "B",
|
||||
"hide": false,
|
||||
"type": "reduce",
|
||||
"datasource": {
|
||||
"uid": "-100",
|
||||
"type": "__expr__"
|
||||
},
|
||||
"conditions": [
|
||||
{
|
||||
"type": "query",
|
||||
"evaluator": {
|
||||
"params": [],
|
||||
"type": "gt"
|
||||
},
|
||||
"operator": {
|
||||
"type": "and"
|
||||
},
|
||||
"query": {
|
||||
"params": [
|
||||
"B"
|
||||
]
|
||||
},
|
||||
"reducer": {
|
||||
"params": [],
|
||||
"type": "last"
|
||||
}
|
||||
}
|
||||
],
|
||||
"reducer": "last",
|
||||
"expression": "A"
|
||||
},
|
||||
"relativeTimeRange": {
|
||||
"from": 600,
|
||||
"to": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasourceUid": "-100",
|
||||
"queryType": "",
|
||||
"model": {
|
||||
"refId": "C",
|
||||
"hide": false,
|
||||
"type": "threshold",
|
||||
"datasource": {
|
||||
"uid": "-100",
|
||||
"type": "__expr__"
|
||||
},
|
||||
"conditions": [
|
||||
{
|
||||
"type": "query",
|
||||
"evaluator": {
|
||||
"params": [
|
||||
0
|
||||
],
|
||||
"type": "gt"
|
||||
},
|
||||
"operator": {
|
||||
"type": "and"
|
||||
},
|
||||
"query": {
|
||||
"params": [
|
||||
"C"
|
||||
]
|
||||
},
|
||||
"reducer": {
|
||||
"params": [],
|
||||
"type": "last"
|
||||
}
|
||||
}
|
||||
],
|
||||
"expression": "B"
|
||||
},
|
||||
"relativeTimeRange": {
|
||||
"from": 600,
|
||||
"to": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions"
|
||||
"github.com/grafana/grafana/pkg/services/datasources"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/org"
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
)
|
||||
|
||||
func TestBacktesting(t *testing.T) {
|
||||
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
DisableLegacyAlerting: true,
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{
|
||||
featuremgmt.FlagAlertingBacktesting,
|
||||
},
|
||||
EnableLog: false,
|
||||
})
|
||||
|
||||
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
|
||||
|
||||
userId := createUser(t, env.SQLStore, user.CreateUserCommand{
|
||||
DefaultOrgRole: string(org.RoleAdmin),
|
||||
Password: "admin",
|
||||
Login: "admin",
|
||||
})
|
||||
|
||||
apiCli := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
|
||||
|
||||
input, err := os.ReadFile(filepath.Join("api_backtesting_data.json"))
|
||||
require.NoError(t, err)
|
||||
var testData map[string]apimodels.BacktestConfig
|
||||
require.NoError(t, json.Unmarshal(input, &testData))
|
||||
|
||||
queryRequest, ok := testData["query"]
|
||||
require.Truef(t, ok, "The data file does not contain a field `query`")
|
||||
|
||||
for _, query := range queryRequest.Data {
|
||||
isExpr, _ := query.IsExpression()
|
||||
if isExpr {
|
||||
continue
|
||||
}
|
||||
t.Logf("Creating a new test data source with UID %s", query.DatasourceUID)
|
||||
dsCmd := &datasources.AddDataSourceCommand{
|
||||
Name: "Backtesting-TestDatasource",
|
||||
Type: "testdata",
|
||||
Access: datasources.DS_ACCESS_PROXY,
|
||||
Uid: query.DatasourceUID,
|
||||
UserId: userId,
|
||||
OrgId: 1,
|
||||
}
|
||||
err := env.Server.HTTPServer.DataSourcesService.AddDataSource(context.Background(), dsCmd)
|
||||
require.NoError(t, err)
|
||||
break
|
||||
}
|
||||
|
||||
t.Run("and request contains data", func(t *testing.T) {
|
||||
t.Run("should accept request", func(t *testing.T) {
|
||||
request, ok := testData["data"]
|
||||
require.Truef(t, ok, "The data file does not contain a field `data`")
|
||||
|
||||
status, body := apiCli.SubmitRuleForBacktesting(t, request)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
var result data.Frame
|
||||
require.NoErrorf(t, json.Unmarshal([]byte(body), &result), "cannot parse response to data frame")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("and request contains query", func(t *testing.T) {
|
||||
t.Run("should accept request with query", func(t *testing.T) {
|
||||
status, body := apiCli.SubmitRuleForBacktesting(t, queryRequest)
|
||||
require.Equalf(t, http.StatusOK, status, "Response: %s", body)
|
||||
var result data.Frame
|
||||
require.NoErrorf(t, json.Unmarshal([]byte(body), &result), "cannot parse response to data frame")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("if user does not have permissions", func(t *testing.T) {
|
||||
if !setting.IsEnterprise {
|
||||
t.Skip("Enterprise-only test")
|
||||
}
|
||||
|
||||
testUserId := createUser(t, env.SQLStore, user.CreateUserCommand{
|
||||
DefaultOrgRole: "",
|
||||
Password: "test",
|
||||
Login: "test",
|
||||
})
|
||||
|
||||
testUserApiCli := newAlertingApiClient(grafanaListedAddr, "test", "test")
|
||||
|
||||
t.Run("fail if can't read rules", func(t *testing.T) {
|
||||
status, body := testUserApiCli.SubmitRuleForBacktesting(t, queryRequest)
|
||||
require.Contains(t, body, accesscontrol.ActionAlertingRuleRead)
|
||||
require.Equalf(t, http.StatusForbidden, status, "Response: %s", body)
|
||||
})
|
||||
|
||||
// access control permissions store
|
||||
permissionsStore := resourcepermissions.NewStore(env.SQLStore)
|
||||
_, err := permissionsStore.SetUserResourcePermission(context.Background(),
|
||||
accesscontrol.GlobalOrgID,
|
||||
accesscontrol.User{ID: testUserId},
|
||||
resourcepermissions.SetResourcePermissionCommand{
|
||||
Actions: []string{
|
||||
accesscontrol.ActionAlertingRuleRead,
|
||||
},
|
||||
Resource: "folders",
|
||||
ResourceID: "*",
|
||||
ResourceAttribute: "uid",
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
testUserApiCli.ReloadCachedPermissions(t)
|
||||
|
||||
t.Run("fail if can't query data sources", func(t *testing.T) {
|
||||
status, body := testUserApiCli.SubmitRuleForBacktesting(t, queryRequest)
|
||||
require.Contains(t, body, "user is not authorized to query one or many data sources used by the rule")
|
||||
require.Equalf(t, http.StatusUnauthorized, status, "Response: %s", body)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -311,3 +311,22 @@ func (a apiClient) GetAllRulesGroupInFolder(t *testing.T, folder string) apimode
|
||||
require.NoError(t, json.Unmarshal(b, &result))
|
||||
return result
|
||||
}
|
||||
|
||||
func (a apiClient) SubmitRuleForBacktesting(t *testing.T, config apimodels.BacktestConfig) (int, string) {
|
||||
t.Helper()
|
||||
buf := bytes.Buffer{}
|
||||
enc := json.NewEncoder(&buf)
|
||||
err := enc.Encode(config)
|
||||
require.NoError(t, err)
|
||||
|
||||
u := fmt.Sprintf("%s/api/v1/rule/backtest", a.url)
|
||||
// nolint:gosec
|
||||
resp, err := http.Post(u, "application/json", &buf)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user