public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugins: Add file store abstraction for handling plugin files (#65432)

Browse Source 
* add file store

* fix markdown fetch bug

* add markdown tests

* fix var name
This commit is contained in:
Will Browne
2023-03-29 12:55:55 +02:00
committed by GitHub
parent 562d8dba5d
commit 7bbe255150
13 changed files with 212 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/http_server.go
+3 -1
View File
@@ -133,6 +133,7 @@ type HTTPServer struct {
pluginClient plugins.Client
pluginStore plugins.Store
pluginInstaller plugins.Installer
pluginFileStore plugins.FileStore
pluginDashboardService plugindashboards.Service
pluginStaticRouteResolver plugins.StaticRouteResolver
pluginErrorResolver plugins.ErrorResolver
@@ -230,7 +231,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
quotaService quota.Service, socialService social.Service, tracer tracing.Tracer,
encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService,
pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service,
dataSourcesService datasources.DataSourceService, queryDataService query.Service,
dataSourcesService datasources.DataSourceService, queryDataService query.Service, pluginFileStore plugins.FileStore,
teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service,
authInfoService login.AuthInfoService, storageService store.StorageService, httpEntityStore httpentitystore.HTTPEntityStore,
notificationService *notifications.NotificationService, dashboardService dashboards.DashboardService,
@@ -271,6 +272,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
pluginStaticRouteResolver: pluginStaticRouteResolver,
pluginDashboardService: pluginDashboardService,
pluginErrorResolver: pluginErrorResolver,
pluginFileStore: pluginFileStore,
grafanaUpdateChecker: grafanaUpdateChecker,
pluginsUpdateChecker: pluginsUpdateChecker,
SettingsProvider: settingsProvider,
pkg/api/plugins.go
+10 -42
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path"
"path/filepath"
@@ -350,7 +349,7 @@ func (hs *HTTPServer) getPluginAssets(c *contextmodel.ReqContext) {
// serveLocalPluginAsset returns the content of a plugin asset file from the local filesystem to the http client.
func (hs *HTTPServer) serveLocalPluginAsset(c *contextmodel.ReqContext, plugin plugins.PluginDTO, assetPath string) {
f, err := plugin.File(assetPath)
f, err := hs.pluginFileStore.File(c.Req.Context(), plugin.ID, assetPath)
if err != nil {
if errors.Is(err, plugins.ErrFileNotExist) {
c.JsonApiErr(404, "Plugin file not found", nil)
@@ -359,17 +358,6 @@ func (hs *HTTPServer) serveLocalPluginAsset(c *contextmodel.ReqContext, plugin p
c.JsonApiErr(500, "Could not open plugin file", err)
return
}
defer func() {
if err = f.Close(); err != nil {
hs.log.Error("Failed to close plugin file", "err", err)
}
}()
fi, err := f.Stat()
if err != nil {
c.JsonApiErr(500, "Plugin file exists but could not open", err)
return
}
if hs.Cfg.Env == setting.Dev {
c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
@@ -377,17 +365,7 @@ func (hs *HTTPServer) serveLocalPluginAsset(c *contextmodel.ReqContext, plugin p
c.Resp.Header().Set("Cache-Control", "public, max-age=3600")
}
if rs, ok := f.(io.ReadSeeker); ok {
http.ServeContent(c.Resp, c.Req, assetPath, fi.ModTime(), rs)
return
}
b, err := io.ReadAll(f)
if err != nil {
c.JsonApiErr(500, "Plugin file exists but could not read", err)
return
}
http.ServeContent(c.Resp, c.Req, assetPath, fi.ModTime(), bytes.NewReader(b))
http.ServeContent(c.Resp, c.Req, assetPath, f.ModTime, bytes.NewReader(f.Content))
}
// redirectCDNPluginAsset redirects the http request to specified asset path on the configured plugins CDN.
@@ -538,34 +516,24 @@ func translatePluginRequestErrorToAPIError(err error) response.Response {
return response.Error(500, "Plugin request failed", err)
}
func (hs *HTTPServer) pluginMarkdown(ctx context.Context, pluginId string, name string) ([]byte, error) {
plugin, exists := hs.pluginStore.Plugin(ctx, pluginId)
if !exists {
return make([]byte, 0), plugins.NotFoundError{PluginID: pluginId}
}
func (hs *HTTPServer) pluginMarkdown(ctx context.Context, pluginID string, name string) ([]byte, error) {
file, err := mdFilepath(strings.ToUpper(name))
if err != nil {
return make([]byte, 0), err
}
md, err := plugin.File(file)
md, err := hs.pluginFileStore.File(ctx, pluginID, file)
if err != nil {
md, err = plugin.File(strings.ToLower(file))
if errors.Is(err, plugins.ErrPluginNotInstalled) {
return make([]byte, 0), plugins.NotFoundError{PluginID: pluginID}
}
md, err = hs.pluginFileStore.File(ctx, pluginID, strings.ToLower(file))
if err != nil {
return make([]byte, 0), nil
}
}
defer func() {
if err = md.Close(); err != nil {
hs.log.Error("Failed to close plugin markdown file", "err", err)
}
}()
d, err := io.ReadAll(md)
if err != nil {
return make([]byte, 0), nil
}
return d, nil
return md.Content, nil
}
func mdFilepath(mdFilename string) (string, error) {
pkg/api/plugins_test.go
+76 -96
View File
@@ -1,13 +1,11 @@
package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
@@ -27,6 +25,9 @@ import (
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/config"
"github.com/grafana/grafana/pkg/plugins/manager/fakes"
"github.com/grafana/grafana/pkg/plugins/manager/filestore"
"github.com/grafana/grafana/pkg/plugins/manager/registry"
"github.com/grafana/grafana/pkg/plugins/manager/store"
"github.com/grafana/grafana/pkg/plugins/pluginscdn"
ac "github.com/grafana/grafana/pkg/services/accesscontrol"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
@@ -140,6 +141,7 @@ func Test_PluginsInstallAndUninstall_AccessControl(t *testing.T) {
PluginAdminExternalManageEnabled: tc.pluginAdminExternalManageEnabled}
hs.orgService = &orgtest.FakeOrgService{ExpectedOrg: &org.Org{}}
hs.pluginInstaller = NewFakePluginInstaller()
hs.pluginFileStore = &fakes.FakePluginFileStore{}
})
t.Run(testName("Install", tc), func(t *testing.T) {
@@ -172,10 +174,10 @@ func Test_GetPluginAssetCDNRedirect(t *testing.T) {
nonCdnPlugin := &plugins.Plugin{
JSONData: plugins.JSONData{ID: nonCDNPluginID, Info: plugins.Info{Version: "2.0.0"}},
}
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{
cdnPlugin.ToDTO(),
nonCdnPlugin.ToDTO(),
registry := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
cdnPluginID: cdnPlugin,
nonCDNPluginID: nonCdnPlugin,
},
}
cfg := setting.NewCfg()
@@ -200,7 +202,7 @@ func Test_GetPluginAssetCDNRedirect(t *testing.T) {
"When calling GET for a CDN plugin on",
fmt.Sprintf("/public/plugins/%s/%s", cdnPluginID, cas.assetURL),
"/public/plugins/:pluginId/*",
cfg, service, func(sc *scenarioContext) {
cfg, registry, func(sc *scenarioContext) {
// Get the prometheus metric (to test that the handler is instrumented correctly)
counter := pluginsCDNFallbackRedirectRequests.With(prometheus.Labels{
"plugin_id": cdnPluginID,
@@ -230,7 +232,7 @@ func Test_GetPluginAssetCDNRedirect(t *testing.T) {
"When calling GET for a non-CDN plugin on",
fmt.Sprintf("/public/plugins/%s/%s", nonCDNPluginID, "module.js"),
"/public/plugins/:pluginId/*",
cfg, service, func(sc *scenarioContext) {
cfg, registry, func(sc *scenarioContext) {
// Here the metric should not increment
var m dto.Metric
counter := pluginsCDNFallbackRedirectRequests.With(prometheus.Labels{
@@ -275,14 +277,16 @@ func Test_GetPluginAssets(t *testing.T) {
requestedFile := filepath.Clean(tmpFile.Name())
t.Run("Given a request for an existing plugin file", func(t *testing.T) {
p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{requestedFile: {}}, filepath.Dir(requestedFile)))
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{p},
p := createPlugin(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{requestedFile: {}}, filepath.Dir(requestedFile)))
pluginRegistry := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
p.ID: p,
},
}
url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*",
setting.NewCfg(), service, func(sc *scenarioContext) {
setting.NewCfg(), pluginRegistry, func(sc *scenarioContext) {
callGetPluginAsset(sc)
require.Equal(t, 200, sc.resp.Code)
@@ -291,14 +295,16 @@ func Test_GetPluginAssets(t *testing.T) {
})
t.Run("Given a request for a relative path", func(t *testing.T) {
p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, ""))
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{p},
p := createPlugin(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, ""))
pluginRegistry := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
p.ID: p,
},
}
url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, tmpFileInParentDir.Name())
pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*",
setting.NewCfg(), service, func(sc *scenarioContext) {
setting.NewCfg(), pluginRegistry, func(sc *scenarioContext) {
callGetPluginAsset(sc)
require.Equal(t, 404, sc.resp.Code)
@@ -306,16 +312,18 @@ func Test_GetPluginAssets(t *testing.T) {
})
t.Run("Given a request for an existing plugin file that is not listed as a signature covered file", func(t *testing.T) {
p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.Core, plugins.NewLocalFS(map[string]struct{}{
p := createPlugin(plugins.JSONData{ID: pluginID}, plugins.Core, plugins.NewLocalFS(map[string]struct{}{
requestedFile: {},
}, ""))
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{p},
pluginRegistry := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
p.ID: p,
},
}
url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*",
setting.NewCfg(), service, func(sc *scenarioContext) {
setting.NewCfg(), pluginRegistry, func(sc *scenarioContext) {
callGetPluginAsset(sc)
require.Equal(t, 200, sc.resp.Code)
@@ -324,9 +332,11 @@ func Test_GetPluginAssets(t *testing.T) {
})
t.Run("Given a request for an non-existing plugin file", func(t *testing.T) {
p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, ""))
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{p},
p := createPlugin(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, ""))
service := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
p.ID: p,
},
}
requestedFile := "nonExistent"
@@ -344,14 +354,10 @@ func Test_GetPluginAssets(t *testing.T) {
})
t.Run("Given a request for an non-existing plugin", func(t *testing.T) {
service := &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{},
}
requestedFile := "nonExistent"
url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*",
setting.NewCfg(), service, func(sc *scenarioContext) {
setting.NewCfg(), fakes.NewFakePluginRegistry(), func(sc *scenarioContext) {
callGetPluginAsset(sc)
var respJson map[string]interface{}
@@ -471,11 +477,12 @@ func TestMakePluginResourceRequestContentTypeEmpty(t *testing.T) {
func TestPluginMarkdown(t *testing.T) {
t.Run("Plugin not installed returns error", func(t *testing.T) {
hs := HTTPServer{
pluginStore: &plugins.FakePluginStore{
PluginList: []plugins.PluginDTO{},
pluginFileStore := &fakes.FakePluginFileStore{
FileFunc: func(ctx context.Context, pluginID, filename string) (*plugins.File, error) {
return nil, plugins.ErrPluginNotInstalled
},
}
hs := HTTPServer{pluginFileStore: pluginFileStore}
pluginID := "test-datasource"
md, err := hs.pluginMarkdown(context.Background(), pluginID, "test")
@@ -485,20 +492,16 @@ func TestPluginMarkdown(t *testing.T) {
t.Run("File fetch will be retried using different casing if error occurs", func(t *testing.T) {
var requestedFiles []string
pluginFiles := &fakes.FakePluginFiles{
OpenFunc: func(name string) (fs.File, error) {
requestedFiles = append(requestedFiles, name)
pluginFileStore := &fakes.FakePluginFileStore{
FileFunc: func(ctx context.Context, pluginID, filename string) (*plugins.File, error) {
requestedFiles = append(requestedFiles, filename)
return nil, errors.New("some error")
},
}
p := createPluginDTO(plugins.JSONData{ID: "test-app"}, plugins.External, pluginFiles)
hs := HTTPServer{pluginFileStore: pluginFileStore}
hs := HTTPServer{
pluginStore: &plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p}},
}
md, err := hs.pluginMarkdown(context.Background(), p.ID, "reAdMe")
md, err := hs.pluginMarkdown(context.Background(), "", "reAdMe")
require.NoError(t, err)
require.Equal(t, []byte{}, md)
require.Equal(t, []string{"README.md", "readme.md"}, requestedFiles)
@@ -524,54 +527,44 @@ func TestPluginMarkdown(t *testing.T) {
}
for _, tc := range tcs {
data := []byte{123}
var requestedFiles []string
pluginFiles := &fakes.FakePluginFiles{
OpenFunc: func(name string) (fs.File, error) {
requestedFiles = append(requestedFiles, name)
return &FakeFile{data: bytes.NewReader(nil)}, nil
pluginFileStore := &fakes.FakePluginFileStore{
FileFunc: func(ctx context.Context, pluginID, filename string) (*plugins.File, error) {
requestedFiles = append(requestedFiles, filename)
return &plugins.File{Content: data}, nil
},
}
p := createPluginDTO(plugins.JSONData{ID: "test-app"}, plugins.External, pluginFiles)
hs := HTTPServer{pluginFileStore: pluginFileStore}
hs := HTTPServer{
pluginStore: &plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p}},
}
md, err := hs.pluginMarkdown(context.Background(), p.ID, tc.filePath)
md, err := hs.pluginMarkdown(context.Background(), "test-datasource", tc.filePath)
require.NoError(t, err)
require.Equal(t, []byte{}, md)
require.Equal(t, data, md)
require.Equal(t, tc.expected, requestedFiles)
}
})
t.Run("Non markdown file request returns an error", func(t *testing.T) {
p := createPluginDTO(plugins.JSONData{ID: "test-app"}, plugins.External, nil)
hs := HTTPServer{
pluginStore: &plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p}},
}
hs := HTTPServer{pluginFileStore: &fakes.FakePluginFileStore{}}
md, err := hs.pluginMarkdown(context.Background(), p.ID, "test.json")
md, err := hs.pluginMarkdown(context.Background(), "", "test.json")
require.ErrorIs(t, err, ErrUnexpectedFileExtension)
require.Equal(t, []byte{}, md)
})
t.Run("Happy path", func(t *testing.T) {
data := []byte{1, 2, 3}
fakeFile := &FakeFile{data: bytes.NewReader(data)}
pluginFiles := &fakes.FakePluginFiles{
OpenFunc: func(name string) (fs.File, error) {
return fakeFile, nil
pluginFileStore := &fakes.FakePluginFileStore{
FileFunc: func(ctx context.Context, pluginID, filename string) (*plugins.File, error) {
return &plugins.File{Content: data}, nil
},
}
p := createPluginDTO(plugins.JSONData{ID: "test-app"}, plugins.External, pluginFiles)
hs := HTTPServer{
pluginStore: &plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p}},
}
hs := HTTPServer{pluginFileStore: pluginFileStore}
md, err := hs.pluginMarkdown(context.Background(), p.ID, "someFile")
md, err := hs.pluginMarkdown(context.Background(), "", "someFile")
require.NoError(t, err)
require.Equal(t, data, md)
})
@@ -582,13 +575,14 @@ func callGetPluginAsset(sc *scenarioContext) {
}
func pluginAssetScenario(t *testing.T, desc string, url string, urlPattern string,
cfg *setting.Cfg, pluginStore plugins.Store, fn scenarioFunc) {
cfg *setting.Cfg, pluginRegistry registry.Service, fn scenarioFunc) {
t.Run(fmt.Sprintf("%s %s", desc, url), func(t *testing.T) {
cfg.IsFeatureToggleEnabled = func(_ string) bool { return false }
hs := HTTPServer{
Cfg: cfg,
pluginStore: pluginStore,
log: log.NewNopLogger(),
Cfg: cfg,
pluginStore: store.New(pluginRegistry),
pluginFileStore: filestore.ProvideService(pluginRegistry),
log: log.NewNopLogger(),
pluginsCDNService: pluginscdn.ProvideService(&config.Cfg{
PluginsCDNURLTemplate: cfg.PluginsCDNURLTemplate,
PluginSettings: cfg.PluginSettings,
@@ -648,19 +642,24 @@ func (c *fakePluginClient) QueryData(ctx context.Context, req *backend.QueryData
}
func Test_PluginsList_AccessControl(t *testing.T) {
p1 := createPluginDTO(plugins.JSONData{
p1 := createPlugin(plugins.JSONData{
ID: "test-app", Type: "app", Name: "test-app",
Info: plugins.Info{
Version: "1.0.0",
}}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, ""))
p2 := createPluginDTO(
p2 := createPlugin(
plugins.JSONData{ID: "mysql", Type: "datasource", Name: "MySQL",
Info: plugins.Info{
Author: plugins.InfoLink{Name: "Grafana Labs", URL: "https://grafana.com"},
Description: "Data source for MySQL databases",
}}, plugins.Core, plugins.NewLocalFS(map[string]struct{}{}, ""))
pluginStore := plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p1, p2}}
pluginRegistry := &fakes.FakePluginRegistry{
Store: map[string]*plugins.Plugin{
p1.ID: p1,
p2.ID: p2,
},
}
pluginSettings := pluginsettings.FakePluginSettings{Plugins: map[string]*pluginsettings.DTO{
"test-app": {ID: 0, OrgID: 1, PluginID: "test-app", PluginVersion: "1.0.0", Enabled: true},
@@ -693,9 +692,10 @@ func Test_PluginsList_AccessControl(t *testing.T) {
server := SetupAPITestServer(t, func(hs *HTTPServer) {
hs.Cfg = setting.NewCfg()
hs.PluginSettings = &pluginSettings
hs.pluginStore = pluginStore
hs.pluginStore = store.New(pluginRegistry)
hs.pluginFileStore = filestore.ProvideService(pluginRegistry)
var err error
hs.pluginsUpdateChecker, err = updatechecker.ProvidePluginsService(hs.Cfg, pluginStore, tracing.InitializeTracerForTest())
hs.pluginsUpdateChecker, err = updatechecker.ProvidePluginsService(hs.Cfg, nil, tracing.InitializeTracerForTest())
require.NoError(t, err)
})
@@ -713,30 +713,10 @@ func Test_PluginsList_AccessControl(t *testing.T) {
}
}
func createPluginDTO(jd plugins.JSONData, class plugins.Class, files plugins.FS) plugins.PluginDTO {
p := &plugins.Plugin{
func createPlugin(jd plugins.JSONData, class plugins.Class, files plugins.FS) *plugins.Plugin {
return &plugins.Plugin{
JSONData: jd,
Class: class,
FS: files,
}
return p.ToDTO()
}
var _ fs.File = (*FakeFile)(nil)
type FakeFile struct {
data io.Reader
}
func (f *FakeFile) Stat() (fs.FileInfo, error) {
return nil, nil
}
func (f *FakeFile) Read(bytes []byte) (int, error) {
return f.data.Read(bytes)
}
func (f *FakeFile) Close() error {
return nil
}