public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugins: Fix flaky backend tests (#113945)

Browse Source 
* fix flaky tests

* tidy
This commit is contained in:
Will Browne
2025-11-14 15:09:12 +00:00
committed by GitHub
parent f7e07d814a
commit 9818822e87
1 changed files with 152 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tests/api/plugins/backendplugin/backendplugin_test.go
+152 -133
View File
@@ -23,6 +23,7 @@ import (
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/log"
"github.com/grafana/grafana/pkg/server"
"github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/tests/testinfra"
"github.com/grafana/grafana/pkg/tests/testsuite"
@@ -38,6 +39,14 @@ func TestMain(m *testing.M) {
func TestIntegrationBackendPlugins(t *testing.T) {
testutil.SkipIntegrationTestInShortMode(t)
// Set up shared Grafana instance once for all test scenarios
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
})
grafanaListeningAddr, testEnv := testinfra.StartGrafanaEnv(t, dir, path)
testEnv.Cfg.LoginCookieName = loginCookieName
oauthToken := &oauth2.Token{
TokenType: "bearer",
AccessToken: "access-token",
@@ -46,7 +55,102 @@ func TestIntegrationBackendPlugins(t *testing.T) {
}
oauthToken = oauthToken.WithExtra(map[string]any{"id_token": "id-token"})
newTestScenario(t, "Datasource with no custom HTTP settings",
// Track scenario number for unique IDs
scenarioNum := 0
newTestScenario := func(name string, opts []testScenarioOption, callback func(t *testing.T, tsCtx *testScenarioContext)) {
scenarioNum++
pluginID := fmt.Sprintf("test-plugin-%d", scenarioNum)
datasourceUID := fmt.Sprintf("test-plugin-%d", scenarioNum)
tsCtx := testScenarioContext{
testPluginID: pluginID,
uid: datasourceUID,
grafanaListeningAddr: grafanaListeningAddr,
testEnv: testEnv,
}
ctx := context.Background()
// Each scenario gets its own outgoing server
tsCtx.outgoingServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tsCtx.outgoingRequest = r
w.WriteHeader(http.StatusUnauthorized)
}))
t.Cleanup(tsCtx.outgoingServer.Close)
// Create plugin
p, bp := createTestPlugin(pluginID, &tsCtx)
tsCtx.backendTestPlugin = bp
err := testEnv.PluginRegistry.Add(ctx, p)
require.NoError(t, err)
// Set up datasource
cmd := &datasources.AddDataSourceCommand{
OrgID: 1,
Access: datasources.DS_ACCESS_PROXY,
Name: fmt.Sprintf("TestPlugin-%d", scenarioNum),
Type: pluginID,
UID: datasourceUID,
URL: tsCtx.outgoingServer.URL,
JsonData: simplejson.New(),
SecureJsonData: map[string]string{},
}
in := &testScenarioInput{ds: cmd}
for _, opt := range opts {
opt(in)
}
tsCtx.modifyIncomingRequest = in.modifyIncomingRequest
if in.modifyCallResourceResponse == nil {
in.modifyCallResourceResponse = func(sender backend.CallResourceResponseSender) error {
responseHeaders := map[string][]string{
"Connection": {"close, TE"},
"Te": {"foo", "bar, trailers"},
"Proxy-Connection": {"should be deleted"},
"Upgrade": {"foo"},
"Set-Cookie": {"should be deleted"},
"X-Custom": {"should not be deleted"},
}
return sender.Send(&backend.CallResourceResponse{
Status: http.StatusOK,
Headers: responseHeaders,
})
}
}
tsCtx.modifyCallResourceResponse = in.modifyCallResourceResponse
if in.token != nil {
testEnv.OAuthTokenService.Token = in.token
} else {
testEnv.OAuthTokenService.Token = nil
}
_, err = testEnv.Server.HTTPServer.DataSourcesService.AddDataSource(ctx, cmd)
require.NoError(t, err)
getDataSourceQuery := &datasources.GetDataSourceQuery{
OrgID: 1,
UID: datasourceUID,
}
dataSource, err := testEnv.Server.HTTPServer.DataSourcesService.GetDataSource(ctx, getDataSourceQuery)
require.NoError(t, err)
rt, err := testEnv.Server.HTTPServer.DataSourcesService.GetHTTPTransport(ctx, dataSource, testEnv.HTTPClientProvider)
require.NoError(t, err)
tsCtx.rt = rt
t.Run(name, func(t *testing.T) {
callback(t, &tsCtx)
})
}
newTestScenario("Datasource with no custom HTTP settings",
options(
withIncomingRequest(func(req *http.Request) {
req.Header.Set("X-Custom", "custom")
@@ -94,7 +198,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
})
})
newTestScenario(t, "Datasource with most HTTP settings set except oauthPassThru and oauth token available",
newTestScenario("Datasource with most HTTP settings set except oauthPassThru and oauth token available",
options(
withIncomingRequest(func(req *http.Request) {
req.AddCookie(&http.Cookie{Name: "cookie1"})
@@ -151,7 +255,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
})
})
newTestScenario(t, "Datasource with oauthPassThru and basic auth configured and oauth token available",
newTestScenario("Datasource with oauthPassThru and basic auth configured and oauth token available",
options(
withOAuthToken(oauthToken),
withDsOAuthForwarding(),
@@ -199,7 +303,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
})
})
newTestScenario(t, "Datasource with resource returning non-default content-type should not be kept",
newTestScenario("Datasource with resource returning non-default content-type should not be kept",
options(
withCallResourceResponse(func(sender backend.CallResourceResponseSender) error {
return sender.Send(&backend.CallResourceResponse{
@@ -221,7 +325,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
})
})
newTestScenario(t, "Datasource with resource returning 204 (no content) status should not set content-type header",
newTestScenario("Datasource with resource returning 204 (no content) status should not set content-type header",
options(
withCallResourceResponse(func(sender backend.CallResourceResponseSender) error {
return sender.Send(&backend.CallResourceResponse{
@@ -236,7 +340,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
})
})
newTestScenario(t, "Datasource with resource returning streaming content should return chunked transfer encoding",
newTestScenario("Datasource with resource returning streaming content should return chunked transfer encoding",
options(
withCallResourceResponse(func(sender backend.CallResourceResponseSender) error {
err := sender.Send(&backend.CallResourceResponse{
@@ -261,13 +365,13 @@ func TestIntegrationBackendPlugins(t *testing.T) {
require.Equal(t, "text/plain", resp.Header.Get("Content-Type"))
require.Equal(t, http.StatusOK, resp.StatusCode)
require.Equal(t, []string{"chunked"}, resp.TransferEncoding)
bytes, err := io.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, "msg 1\r\nmsg 2\r\n", string(bytes))
require.Equal(t, "msg 1\r\nmsg 2\r\n", string(body))
})
})
newTestScenario(t, "Query data error should return expected status code and marked with downstream status",
newTestScenario("Query data error should return expected status code and marked with downstream status",
options(),
func(t *testing.T, tsCtx *testScenarioContext) {
tsCtx.backendTestPlugin.QueryDataHandler = backend.QueryDataHandlerFunc(func(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -283,12 +387,12 @@ func TestIntegrationBackendPlugins(t *testing.T) {
rmd := requestmeta.GetRequestMetaData(tsCtx.incomingRequest.Context())
require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
})
newTestScenario(t, "Call resource error should return expected status code and marked with downstream status",
newTestScenario("Call resource error should return expected status code and marked with downstream status",
options(),
func(t *testing.T, tsCtx *testScenarioContext) {
tsCtx.backendTestPlugin.CallResourceHandler = backend.CallResourceHandlerFunc(func(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
@@ -300,16 +404,16 @@ func TestIntegrationBackendPlugins(t *testing.T) {
require.NoError(t, err)
require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
require.NotNil(t, tsCtx.incomingRequest)
require.Equal(t, "/api/datasources/uid/test-plugin/resources", tsCtx.incomingRequest.URL.Path)
require.Equal(t, fmt.Sprintf("/api/datasources/uid/%s/resources", tsCtx.uid), tsCtx.incomingRequest.URL.Path)
rmd := requestmeta.GetRequestMetaData(tsCtx.incomingRequest.Context())
require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
})
newTestScenario(t, "Check health error should return expected status code and marked with downstream status",
newTestScenario("Check health error should return expected status code and marked with downstream status",
options(),
func(t *testing.T, tsCtx *testScenarioContext) {
tsCtx.backendTestPlugin.CheckHealthHandler = backend.CheckHealthHandlerFunc(func(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
@@ -321,16 +425,16 @@ func TestIntegrationBackendPlugins(t *testing.T) {
require.NoError(t, err)
require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
require.NotNil(t, tsCtx.incomingRequest)
require.Equal(t, "/api/datasources/uid/test-plugin/health", tsCtx.incomingRequest.URL.Path)
require.Equal(t, fmt.Sprintf("/api/datasources/uid/%s/health", tsCtx.uid), tsCtx.incomingRequest.URL.Path)
rmd := requestmeta.GetRequestMetaData(tsCtx.incomingRequest.Context())
require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
})
newTestScenario(t, "Call resource response with 502 status code should be marked with downstream status",
newTestScenario("Call resource response with 502 status code should be marked with downstream status",
options(),
func(t *testing.T, tsCtx *testScenarioContext) {
tsCtx.backendTestPlugin.CallResourceHandler = backend.CallResourceHandlerFunc(func(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
@@ -347,16 +451,16 @@ func TestIntegrationBackendPlugins(t *testing.T) {
require.NoError(t, err)
require.Equal(t, http.StatusBadGateway, resp.StatusCode, string(b))
require.NotNil(t, tsCtx.incomingRequest)
require.Equal(t, "/api/datasources/uid/test-plugin/resources", tsCtx.incomingRequest.URL.Path)
require.Equal(t, fmt.Sprintf("/api/datasources/uid/%s/resources", tsCtx.uid), tsCtx.incomingRequest.URL.Path)
rmd := requestmeta.GetRequestMetaData(tsCtx.incomingRequest.Context())
require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
})
newTestScenario(t, "Query data response that includes a query data response error should return expected status code and marked with downstream status",
newTestScenario("Query data response that includes a query data response error should return expected status code and marked with downstream status",
options(),
func(t *testing.T, tsCtx *testScenarioContext) {
tsCtx.backendTestPlugin.QueryDataHandler = backend.QueryDataHandlerFunc(func(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -379,7 +483,7 @@ func TestIntegrationBackendPlugins(t *testing.T) {
rmd := requestmeta.GetRequestMetaData(tsCtx.incomingRequest.Context())
require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
})
@@ -461,102 +565,6 @@ func withCallResourceResponse(cb func(sender backend.CallResourceResponseSender)
}
}
func newTestScenario(t *testing.T, name string, opts []testScenarioOption, callback func(t *testing.T, ctx *testScenarioContext)) {
tsCtx := testScenarioContext{
testPluginID: "test-plugin",
}
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableAnonymous: true,
// EnableLog: true,
})
grafanaListeningAddr, testEnv := testinfra.StartGrafanaEnv(t, dir, path)
testEnv.RequestMiddleware = func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tsCtx.incomingRequest = r
next.ServeHTTP(w, r)
})
}
tsCtx.grafanaListeningAddr = grafanaListeningAddr
testEnv.Cfg.LoginCookieName = loginCookieName
tsCtx.testEnv = testEnv
ctx := context.Background()
tsCtx.outgoingServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tsCtx.outgoingRequest = r
w.WriteHeader(http.StatusUnauthorized)
}))
t.Cleanup(tsCtx.outgoingServer.Close)
testPlugin, backendTestPlugin := createTestPlugin(tsCtx.testPluginID)
tsCtx.backendTestPlugin = backendTestPlugin
err := testEnv.PluginRegistry.Add(ctx, testPlugin)
require.NoError(t, err)
jsonData := simplejson.New()
secureJSONData := map[string]string{}
tsCtx.uid = "test-plugin"
cmd := &datasources.AddDataSourceCommand{
OrgID: 1,
Access: datasources.DS_ACCESS_PROXY,
Name: "TestPlugin",
Type: tsCtx.testPluginID,
UID: tsCtx.uid,
URL: tsCtx.outgoingServer.URL,
JsonData: jsonData,
SecureJsonData: secureJSONData,
}
in := &testScenarioInput{ds: cmd}
for _, opt := range opts {
opt(in)
}
tsCtx.modifyIncomingRequest = in.modifyIncomingRequest
if in.modifyCallResourceResponse == nil {
in.modifyCallResourceResponse = func(sender backend.CallResourceResponseSender) error {
responseHeaders := map[string][]string{
"Connection": {"close, TE"},
"Te": {"foo", "bar, trailers"},
"Proxy-Connection": {"should be deleted"},
"Upgrade": {"foo"},
"Set-Cookie": {"should be deleted"},
"X-Custom": {"should not be deleted"},
}
return sender.Send(&backend.CallResourceResponse{
Status: http.StatusOK,
Headers: responseHeaders,
})
}
}
tsCtx.modifyCallResourceResponse = in.modifyCallResourceResponse
tsCtx.testEnv.OAuthTokenService.Token = in.token
_, err = testEnv.Server.HTTPServer.DataSourcesService.AddDataSource(ctx, cmd)
require.NoError(t, err)
getDataSourceQuery := &datasources.GetDataSourceQuery{
OrgID: 1,
UID: tsCtx.uid,
}
dataSource, err := testEnv.Server.HTTPServer.DataSourcesService.GetDataSource(ctx, getDataSourceQuery)
require.NoError(t, err)
rt, err := testEnv.Server.HTTPServer.DataSourcesService.GetHTTPTransport(ctx, dataSource, testEnv.HTTPClientProvider)
require.NoError(t, err)
tsCtx.rt = rt
t.Run(name, func(t *testing.T) {
callback(t, &tsCtx)
})
}
func createRegularQuery(t *testing.T, tsCtx *testScenarioContext) dtos.MetricRequest {
t.Helper()
@@ -630,7 +638,7 @@ func (tsCtx *testScenarioContext) runQueryDataTest(t *testing.T, mr dtos.MetricR
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
_, err = io.ReadAll(resp.Body)
@@ -659,7 +667,7 @@ func (tsCtx *testScenarioContext) runCheckHealthTest(t *testing.T, callback func
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
if err = resp.Body.Close(); err != nil {
tsCtx.testEnv.Server.HTTPServer.Cfg.Logger.Error("Failed to close body", "error", err)
}
}()
@@ -688,7 +696,7 @@ func (tsCtx *testScenarioContext) runCheckHealthTest(t *testing.T, callback func
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
_, err = io.ReadAll(resp.Body)
@@ -723,7 +731,7 @@ func (tsCtx *testScenarioContext) runCallResourceTest(t *testing.T, callback fun
return err
}
defer func() {
if err := resp.Body.Close(); err != nil {
if err = resp.Body.Close(); err != nil {
tsCtx.testEnv.Server.HTTPServer.Cfg.Logger.Error("Failed to close body", "error", err)
}
}()
@@ -750,7 +758,7 @@ func (tsCtx *testScenarioContext) runCallResourceTest(t *testing.T, callback fun
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
err = resp.Body.Close()
require.NoError(t, err)
})
@@ -809,7 +817,7 @@ func createQueryDataHTTPRequest(t *testing.T, tsCtx *testScenarioContext, mr dto
return req
}
func createTestPlugin(id string) (*plugins.Plugin, *testPlugin) {
func createTestPlugin(id string, tsCtx *testScenarioContext) (*plugins.Plugin, *testPlugin) {
p := &plugins.Plugin{
JSONData: plugins.JSONData{
ID: id,
@@ -819,8 +827,9 @@ func createTestPlugin(id string) (*plugins.Plugin, *testPlugin) {
p.SetLogger(log.New("test-plugin"))
tp := &testPlugin{
pluginID: id,
logger: p.Logger(),
pluginID: id,
logger: p.Logger(),
scenarioCtx: tsCtx,
QueryDataHandler: backend.QueryDataHandlerFunc(func(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
return &backend.QueryDataResponse{}, nil
}),
@@ -831,8 +840,9 @@ func createTestPlugin(id string) (*plugins.Plugin, *testPlugin) {
}
type testPlugin struct {
pluginID string
logger log.Logger
pluginID string
logger log.Logger
scenarioCtx *testScenarioContext
backend.CheckHealthHandler
backend.CallResourceHandler
backend.QueryDataHandler
@@ -882,6 +892,10 @@ func (tp *testPlugin) CollectMetrics(_ context.Context, _ *backend.CollectMetric
}
func (tp *testPlugin) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil {
tp.scenarioCtx.incomingRequest = reqCtx.Req
}
if tp.CheckHealthHandler != nil {
return tp.CheckHealthHandler.CheckHealth(ctx, req)
}
@@ -890,6 +904,10 @@ func (tp *testPlugin) CheckHealth(ctx context.Context, req *backend.CheckHealthR
}
func (tp *testPlugin) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil {
tp.scenarioCtx.incomingRequest = reqCtx.Req
}
if tp.QueryDataHandler != nil {
return tp.QueryDataHandler.QueryData(ctx, req)
}
@@ -898,6 +916,10 @@ func (tp *testPlugin) QueryData(ctx context.Context, req *backend.QueryDataReque
}
func (tp *testPlugin) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil {
tp.scenarioCtx.incomingRequest = reqCtx.Req
}
if tp.CallResourceHandler != nil {
return tp.CallResourceHandler.CallResource(ctx, req, sender)
}
@@ -926,7 +948,6 @@ func (tp *testPlugin) RunStream(ctx context.Context, req *backend.RunStreamReque
return plugins.ErrMethodNotImplemented
}
// ValidateAdmission implements backend.AdmissionHandler.
func (tp *testPlugin) ValidateAdmission(ctx context.Context, req *backend.AdmissionRequest) (*backend.ValidationResponse, error) {
if tp.AdmissionHandler != nil {
return tp.AdmissionHandler.ValidateAdmission(ctx, req)
@@ -935,7 +956,6 @@ func (tp *testPlugin) ValidateAdmission(ctx context.Context, req *backend.Admiss
return nil, plugins.ErrMethodNotImplemented
}
// MutateAdmission implements backend.AdmissionHandler.
func (tp *testPlugin) MutateAdmission(ctx context.Context, req *backend.AdmissionRequest) (*backend.MutationResponse, error) {
if tp.AdmissionHandler != nil {
return tp.AdmissionHandler.MutateAdmission(ctx, req)
@@ -944,7 +964,6 @@ func (tp *testPlugin) MutateAdmission(ctx context.Context, req *backend.Admissio
return nil, plugins.ErrMethodNotImplemented
}
// ConvertObject implements backend.AdmissionHandler.
func (tp *testPlugin) ConvertObjects(ctx context.Context, req *backend.ConversionRequest) (*backend.ConversionResponse, error) {
if tp.ConversionHandler != nil {
return tp.ConversionHandler.ConvertObjects(ctx, req)
@@ -956,10 +975,10 @@ func (tp *testPlugin) ConvertObjects(ctx context.Context, req *backend.Conversio
func metricRequestWithQueries(t *testing.T, rawQueries ...string) dtos.MetricRequest {
t.Helper()
queries := make([]*simplejson.Json, 0)
for _, q := range rawQueries {
json, err := simplejson.NewJson([]byte(q))
for _, rq := range rawQueries {
q, err := simplejson.NewJson([]byte(rq))
require.NoError(t, err)
queries = append(queries, json)
queries = append(queries, q)
}
return dtos.MetricRequest{
From: "now-1h",