public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugins: Refactor loader + finder to support multiple sourcing methods (#64735)

Browse Source 
* it's cdn time

* tidy body closing

* auto signed

* fix close

* update log name

* remove comments
This commit is contained in:
Will Browne
2023-03-20 14:35:49 +01:00
committed by GitHub
parent eba2c7b522
commit ee2dd62a1f
25 changed files with 389 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/plugins/manager/loader/finder/finder.go
-44
View File
@@ -1,44 +0,0 @@
package finder
import (
"context"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/log"
)
type Service struct {
local *FS
log log.Logger
}
func NewService() *Service {
logger := log.New("plugin.finder")
return &Service{
local: newFS(logger),
log: logger,
}
}
func (f *Service) Find(ctx context.Context, pluginPaths ...string) ([]*plugins.FoundBundle, error) {
if len(pluginPaths) == 0 {
return []*plugins.FoundBundle{}, nil
}
fbs := make(map[string][]*plugins.FoundBundle)
for _, path := range pluginPaths {
local, err := f.local.Find(ctx, path)
if err != nil {
f.log.Warn("Error occurred when trying to find plugin", "path", path)
continue
}
fbs[path] = local
}
var found []*plugins.FoundBundle
for _, fb := range fbs {
found = append(found, fb...)
}
return found, nil
}
pkg/plugins/manager/loader/finder/ifaces.go
+1 -1
View File
@@ -7,5 +7,5 @@ import (
)
type Finder interface {
Find(ctx context.Context, uris ...string) ([]*plugins.FoundBundle, error)
Find(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
}
pkg/plugins/manager/loader/finder/fs.go → pkg/plugins/manager/loader/finder/local.go
+59 -82
View File
@@ -2,9 +2,9 @@ package finder
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
@@ -13,7 +13,6 @@ import (
"github.com/grafana/grafana/pkg/infra/fs"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/log"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/util"
)
@@ -24,32 +23,34 @@ var (
ErrInvalidPluginJSONFilePath = errors.New("invalid plugin.json filepath was provided")
)
type FS struct {
type Local struct {
log log.Logger
}
func newFS(logger log.Logger) *FS {
return &FS{log: logger.New("fs")}
func NewLocalFinder() *Local {
return &Local{
log: log.New("local.finder"),
}
}
func (f *FS) Find(_ context.Context, pluginPaths ...string) ([]*plugins.FoundBundle, error) {
if len(pluginPaths) == 0 {
func (l *Local) Find(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error) {
if len(src.PluginURIs(ctx)) == 0 {
return []*plugins.FoundBundle{}, nil
}
var pluginJSONPaths []string
for _, path := range pluginPaths {
for _, path := range src.PluginURIs(ctx) {
exists, err := fs.Exists(path)
if err != nil {
f.log.Warn("Skipping finding plugins as an error occurred", "path", path, "err", err)
l.log.Warn("Skipping finding plugins as an error occurred", "path", path, "err", err)
continue
}
if !exists {
f.log.Warn("Skipping finding plugins as directory does not exist", "path", path)
l.log.Warn("Skipping finding plugins as directory does not exist", "path", path)
continue
}
paths, err := f.getAbsPluginJSONPaths(path)
paths, err := l.getAbsPluginJSONPaths(path)
if err != nil {
return nil, err
}
@@ -59,20 +60,20 @@ func (f *FS) Find(_ context.Context, pluginPaths ...string) ([]*plugins.FoundBun
// load plugin.json files and map directory to JSON data
foundPlugins := make(map[string]plugins.JSONData)
for _, pluginJSONPath := range pluginJSONPaths {
plugin, err := f.readPluginJSON(pluginJSONPath)
plugin, err := l.readPluginJSON(pluginJSONPath)
if err != nil {
f.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err)
l.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err)
continue
}
pluginJSONAbsPath, err := filepath.Abs(pluginJSONPath)
if err != nil {
f.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginID", plugin.ID, "err", err)
l.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginID", plugin.ID, "err", err)
continue
}
if _, dupe := foundPlugins[filepath.Dir(pluginJSONAbsPath)]; dupe {
f.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", plugin.ID)
l.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", plugin.ID)
continue
}
foundPlugins[filepath.Dir(pluginJSONAbsPath)] = plugin
@@ -121,7 +122,30 @@ func (f *FS) Find(_ context.Context, pluginPaths ...string) ([]*plugins.FoundBun
return result, nil
}
func (f *FS) getAbsPluginJSONPaths(path string) ([]string, error) {
func (l *Local) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) {
reader, err := l.readFile(pluginJSONPath)
defer func() {
if reader == nil {
return
}
if err = reader.Close(); err != nil {
l.log.Warn("Failed to close plugin JSON file", "path", pluginJSONPath, "err", err)
}
}()
if err != nil {
l.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err)
return plugins.JSONData{}, err
}
plugin, err := ReadPluginJSON(reader)
if err != nil {
l.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err)
return plugins.JSONData{}, err
}
return plugin, nil
}
func (l *Local) getAbsPluginJSONPaths(path string) ([]string, error) {
var pluginJSONPaths []string
var err error
@@ -134,11 +158,11 @@ func (f *FS) getAbsPluginJSONPaths(path string) ([]string, error) {
func(currentPath string, fi os.FileInfo, err error) error {
if err != nil {
if errors.Is(err, os.ErrNotExist) {
f.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "err", err)
l.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "err", err)
return nil
}
if errors.Is(err, os.ErrPermission) {
f.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "err", err)
l.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "err", err)
return nil
}
@@ -166,70 +190,6 @@ func (f *FS) getAbsPluginJSONPaths(path string) ([]string, error) {
return pluginJSONPaths, nil
}
func (f *FS) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) {
f.log.Debug("Loading plugin", "path", pluginJSONPath)
if !strings.EqualFold(filepath.Ext(pluginJSONPath), ".json") {
return plugins.JSONData{}, ErrInvalidPluginJSONFilePath
}
absPluginJSONPath, err := filepath.Abs(pluginJSONPath)
if err != nil {
return plugins.JSONData{}, err
}
// Wrapping in filepath.Clean to properly handle
// gosec G304 Potential file inclusion via variable rule.
reader, err := os.Open(filepath.Clean(absPluginJSONPath))
if err != nil {
return plugins.JSONData{}, err
}
defer func() {
if reader == nil {
return
}
if err = reader.Close(); err != nil {
f.log.Warn("Failed to close JSON file", "path", pluginJSONPath, "err", err)
}
}()
plugin := plugins.JSONData{}
if err = json.NewDecoder(reader).Decode(&plugin); err != nil {
return plugins.JSONData{}, err
}
if err = validatePluginJSON(plugin); err != nil {
return plugins.JSONData{}, err
}
if plugin.ID == "grafana-piechart-panel" {
plugin.Name = "Pie Chart (old)"
}
if len(plugin.Dependencies.Plugins) == 0 {
plugin.Dependencies.Plugins = []plugins.Dependency{}
}
if plugin.Dependencies.GrafanaVersion == "" {
plugin.Dependencies.GrafanaVersion = "*"
}
for _, include := range plugin.Includes {
if include.Role == "" {
include.Role = org.RoleViewer
}
}
return plugin, nil
}
func validatePluginJSON(data plugins.JSONData) error {
if data.ID == "" || !data.Type.IsValid() {
return ErrInvalidPluginJSON
}
return nil
}
func collectFilesWithin(dir string) (map[string]struct{}, error) {
files := map[string]struct{}{}
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
@@ -284,3 +244,20 @@ func collectFilesWithin(dir string) (map[string]struct{}, error) {
return files, err
}
func (l *Local) readFile(pluginJSONPath string) (io.ReadCloser, error) {
l.log.Debug("Loading plugin", "path", pluginJSONPath)
if !strings.EqualFold(filepath.Ext(pluginJSONPath), ".json") {
return nil, ErrInvalidPluginJSONFilePath
}
absPluginJSONPath, err := filepath.Abs(pluginJSONPath)
if err != nil {
return nil, err
}
// Wrapping in filepath.Clean to properly handle
// gosec G304 Potential file inclusion via variable rule.
return os.Open(filepath.Clean(absPluginJSONPath))
}
pkg/plugins/manager/loader/finder/fs_test.go → pkg/plugins/manager/loader/finder/local_test.go
+19 -19
View File
@@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/log"
"github.com/grafana/grafana/pkg/plugins/manager/fakes"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/util"
)
@@ -278,8 +278,12 @@ func TestFinder_Find(t *testing.T) {
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
f := newFS(log.NewTestLogger())
pluginBundles, err := f.Find(context.Background(), tc.pluginDirs...)
f := NewLocalFinder()
pluginBundles, err := f.Find(context.Background(), &fakes.FakePluginSource{
PluginURIsFunc: func(ctx context.Context) []string {
return tc.pluginDirs
},
})
if (err != nil) && !errors.Is(err, tc.err) {
t.Errorf("Find() error = %v, expected error %v", err, tc.err)
return
@@ -307,7 +311,7 @@ func TestFinder_getAbsPluginJSONPaths(t *testing.T) {
walk = origWalk
})
finder := newFS(log.NewTestLogger())
finder := NewLocalFinder()
paths, err := finder.getAbsPluginJSONPaths("test")
require.NoError(t, err)
require.Empty(t, paths)
@@ -322,7 +326,7 @@ func TestFinder_getAbsPluginJSONPaths(t *testing.T) {
walk = origWalk
})
finder := newFS(log.NewTestLogger())
finder := NewLocalFinder()
paths, err := finder.getAbsPluginJSONPaths("test")
require.NoError(t, err)
require.Empty(t, paths)
@@ -337,7 +341,7 @@ func TestFinder_getAbsPluginJSONPaths(t *testing.T) {
walk = origWalk
})
finder := newFS(log.NewTestLogger())
finder := NewLocalFinder()
paths, err := finder.getAbsPluginJSONPaths("test")
require.Error(t, err)
require.Empty(t, paths)
@@ -396,7 +400,7 @@ func TestFinder_readPluginJSON(t *testing.T) {
name string
pluginPath string
expected plugins.JSONData
failed bool
err error
}{
{
name: "Valid plugin",
@@ -444,27 +448,23 @@ func TestFinder_readPluginJSON(t *testing.T) {
},
{
name: "Invalid plugin JSON",
pluginPath: "../testdata/invalid-plugin-json/plugin.json",
failed: true,
},
{
name: "Non-existing JSON file",
pluginPath: "nonExistingFile.json",
failed: true,
pluginPath: "../../testdata/invalid-plugin-json/plugin.json",
err: ErrInvalidPluginJSON,
},
}
f := newFS(log.NewTestLogger())
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := f.readPluginJSON(tt.pluginPath)
if (err != nil) && !tt.failed {
t.Errorf("readPluginJSON() error = %v, failed %v", err, tt.failed)
return
reader, err := os.Open(tt.pluginPath)
require.NoError(t, err)
got, err := ReadPluginJSON(reader)
if tt.err != nil {
require.ErrorIs(t, err, tt.err)
}
if !cmp.Equal(got, tt.expected) {
t.Errorf("Unexpected pluginJSONData: %v", cmp.Diff(got, tt.expected))
}
require.NoError(t, reader.Close())
})
}
}
pkg/plugins/manager/loader/finder/util.go
+47
View File
@@ -0,0 +1,47 @@
package finder
import (
"encoding/json"
"io"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/org"
)
func ReadPluginJSON(reader io.Reader) (plugins.JSONData, error) {
plugin := plugins.JSONData{}
if err := json.NewDecoder(reader).Decode(&plugin); err != nil {
return plugins.JSONData{}, err
}
if err := validatePluginJSON(plugin); err != nil {
return plugins.JSONData{}, err
}
if plugin.ID == "grafana-piechart-panel" {
plugin.Name = "Pie Chart (old)"
}
if len(plugin.Dependencies.Plugins) == 0 {
plugin.Dependencies.Plugins = []plugins.Dependency{}
}
if plugin.Dependencies.GrafanaVersion == "" {
plugin.Dependencies.GrafanaVersion = "*"
}
for _, include := range plugin.Includes {
if include.Role == "" {
include.Role = org.RoleViewer
}
}
return plugin, nil
}
func validatePluginJSON(data plugins.JSONData) error {
if data.ID == "" || !data.Type.IsValid() {
return ErrInvalidPluginJSON
}
return nil
}