Plugins: Move discovery logic to plugin sources (#106911)

* move finder behaviour to source

* tidy

* undo go.mod changes

* fix comment

* tidy unsafe local source

pkg/plugins/manager/sources/source_local_disk.go

@@ -3,22 +3,51 @@ package sources
 import (
 	"context"
 	"errors"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 
+	"github.com/grafana/grafana/pkg/infra/fs"
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+var walk = util.Walk
+
+var (
+	ErrInvalidPluginJSONFilePath = errors.New("invalid plugin.json filepath was provided")
 )
 
 type LocalSource struct {
-	paths []string
-	class plugins.Class
+	paths      []string
+	class      plugins.Class
+	strictMode bool // If true, tracks files via a StaticFS
+	log        log.Logger
 }
 
+// NewLocalSource represents a plugin with a fixed set of files.
 func NewLocalSource(class plugins.Class, paths []string) *LocalSource {
 	return &LocalSource{
-		class: class,
-		paths: paths,
+		paths:      paths,
+		class:      class,
+		strictMode: true,
+		log:        log.New("local.source"),
+	}
+}
+
+// NewUnsafeLocalSource represents a plugin that has an unbounded set of files. This useful when running in
+// dev mode whilst developing a plugin.
+func NewUnsafeLocalSource(class plugins.Class, paths []string) *LocalSource {
+	return &LocalSource{
+		paths:      paths,
+		class:      class,
+		strictMode: false,
+		log:        log.New("local.source"),
 	}
 }
 
@@ -26,7 +55,9 @@ func (s *LocalSource) PluginClass(_ context.Context) plugins.Class {
 	return s.class
 }
 
-func (s *LocalSource) PluginURIs(_ context.Context) []string {
+// Paths returns the file system paths that this source will search for plugins.
+// This method is primarily intended for testing purposes.
+func (s *LocalSource) Paths() []string {
 	return s.paths
 }
 
@@ -41,7 +72,189 @@ func (s *LocalSource) DefaultSignature(_ context.Context, _ string) (plugins.Sig
 	}
 }
 
-func DirAsLocalSources(pluginsPath string, class plugins.Class) ([]*LocalSource, error) {
+func (s *LocalSource) Discover(_ context.Context) ([]*plugins.FoundBundle, error) {
+	if len(s.paths) == 0 {
+		return []*plugins.FoundBundle{}, nil
+	}
+
+	pluginJSONPaths := make([]string, 0, len(s.paths))
+	for _, path := range s.paths {
+		exists, err := fs.Exists(path)
+		if err != nil {
+			s.log.Warn("Skipping finding plugins as an error occurred", "path", path, "error", err)
+			continue
+		}
+		if !exists {
+			s.log.Warn("Skipping finding plugins as directory does not exist", "path", path)
+			continue
+		}
+
+		paths, err := s.getAbsPluginJSONPaths(path)
+		if err != nil {
+			return nil, err
+		}
+		pluginJSONPaths = append(pluginJSONPaths, paths...)
+	}
+
+	// load plugin.json files and map directory to JSON data
+	foundPlugins := make(map[string]plugins.JSONData)
+	for _, pluginJSONPath := range pluginJSONPaths {
+		plugin, err := s.readPluginJSON(pluginJSONPath)
+		if err != nil {
+			s.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "error", err)
+			continue
+		}
+
+		pluginJSONAbsPath, err := filepath.Abs(pluginJSONPath)
+		if err != nil {
+			s.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginId", plugin.ID, "error", err)
+			continue
+		}
+
+		foundPlugins[filepath.Dir(pluginJSONAbsPath)] = plugin
+	}
+
+	res := make(map[string]*plugins.FoundBundle)
+	for pluginDir, data := range foundPlugins {
+		var pluginFs plugins.FS
+		pluginFs = plugins.NewLocalFS(pluginDir)
+		if s.strictMode {
+			// Tighten up security by allowing access only to the files present up to this point.
+			// Any new file "sneaked in" won't be allowed and will act as if the file does not exist.
+			var err error
+			pluginFs, err = plugins.NewStaticFS(pluginFs)
+			if err != nil {
+				return nil, err
+			}
+		}
+		res[pluginDir] = &plugins.FoundBundle{
+			Primary: plugins.FoundPlugin{
+				JSONData: data,
+				FS:       pluginFs,
+			},
+		}
+	}
+
+	// Track child plugins and add them to their parent.
+	childPlugins := make(map[string]struct{})
+	for dir, p := range res {
+		// Check if this plugin is the parent of another plugin.
+		for dir2, p2 := range res {
+			if dir == dir2 {
+				continue
+			}
+
+			relPath, err := filepath.Rel(dir, dir2)
+			if err != nil {
+				s.log.Error("Cannot calculate relative path. Skipping", "pluginId", p2.Primary.JSONData.ID, "err", err)
+				continue
+			}
+			if !strings.Contains(relPath, "..") {
+				child := p2.Primary
+				s.log.Debug("Adding child", "parent", p.Primary.JSONData.ID, "child", child.JSONData.ID, "relPath", relPath)
+				p.Children = append(p.Children, &child)
+				childPlugins[dir2] = struct{}{}
+			}
+		}
+	}
+
+	// Remove child plugins from the result (they are already tracked via their parent).
+	result := make([]*plugins.FoundBundle, 0, len(res))
+	for k := range res {
+		if _, ok := childPlugins[k]; !ok {
+			result = append(result, res[k])
+		}
+	}
+
+	return result, nil
+}
+
+func (s *LocalSource) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) {
+	reader, err := s.readFile(pluginJSONPath)
+	defer func() {
+		if reader == nil {
+			return
+		}
+		if err = reader.Close(); err != nil {
+			s.log.Warn("Failed to close plugin JSON file", "path", pluginJSONPath, "error", err)
+		}
+	}()
+	if err != nil {
+		s.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "error", err)
+		return plugins.JSONData{}, err
+	}
+	plugin, err := plugins.ReadPluginJSON(reader)
+	if err != nil {
+		s.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "error", err)
+		return plugins.JSONData{}, err
+	}
+
+	return plugin, nil
+}
+
+func (s *LocalSource) getAbsPluginJSONPaths(path string) ([]string, error) {
+	var pluginJSONPaths []string
+
+	var err error
+	path, err = filepath.Abs(path)
+	if err != nil {
+		return []string{}, err
+	}
+
+	if err = walk(path, true, true,
+		func(currentPath string, fi os.FileInfo, err error) error {
+			if err != nil {
+				if errors.Is(err, os.ErrNotExist) {
+					s.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "error", err)
+					return nil
+				}
+				if errors.Is(err, os.ErrPermission) {
+					s.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "error", err)
+					return nil
+				}
+
+				return fmt.Errorf("filepath.Walk reported an error for %q: %w", currentPath, err)
+			}
+
+			if fi.Name() == "node_modules" {
+				return util.ErrWalkSkipDir
+			}
+
+			if fi.IsDir() {
+				return nil
+			}
+
+			if fi.Name() != "plugin.json" {
+				return nil
+			}
+
+			pluginJSONPaths = append(pluginJSONPaths, currentPath)
+			return nil
+		}); err != nil {
+		return []string{}, err
+	}
+
+	return pluginJSONPaths, nil
+}
+
+func (s *LocalSource) readFile(pluginJSONPath string) (io.ReadCloser, error) {
+	s.log.Debug("Loading plugin", "path", pluginJSONPath)
+
+	if !strings.EqualFold(filepath.Ext(pluginJSONPath), ".json") {
+		return nil, ErrInvalidPluginJSONFilePath
+	}
+
+	absPluginJSONPath, err := filepath.Abs(pluginJSONPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Wrapping in filepath.Clean to properly handle
+	// gosec G304 Potential file inclusion via variable rule.
+	return os.Open(filepath.Clean(absPluginJSONPath))
+}
+
+func DirAsLocalSources(cfg *config.PluginManagementCfg, pluginsPath string, class plugins.Class) ([]*LocalSource, error) {
 	if pluginsPath == "" {
 		return []*LocalSource{}, errors.New("plugins path not configured")
 	}
@@ -64,7 +277,11 @@ func DirAsLocalSources(pluginsPath string, class plugins.Class) ([]*LocalSource,
 
 	sources := make([]*LocalSource, len(pluginDirs))
 	for i, dir := range pluginDirs {
-		sources[i] = NewLocalSource(class, []string{dir})
+		if cfg.DevMode {
+			sources[i] = NewUnsafeLocalSource(class, []string{dir})
+		} else {
+			sources[i] = NewLocalSource(class, []string{dir})
+		}
 	}
 
 	return sources, nil

pkg/plugins/manager/sources/source_local_disk_test.go

@@ -5,17 +5,23 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
 )
 
+var compareOpts = []cmp.Option{cmpopts.IgnoreFields(LocalSource{}, "log"), cmp.AllowUnexported(LocalSource{})}
+
 func TestDirAsLocalSources(t *testing.T) {
 	testdataDir := "../testdata"
 
 	tests := []struct {
 		name        string
 		pluginsPath string
+		cfg         *config.PluginManagementCfg
 		expected    []*LocalSource
 		err         error
 	}{
@@ -28,44 +34,79 @@ func TestDirAsLocalSources(t *testing.T) {
 		{
 			name:        "Directory with subdirectories",
 			pluginsPath: filepath.Join(testdataDir, "pluginRootWithDist"),
+			cfg:         &config.PluginManagementCfg{},
 			expected: []*LocalSource{
 				{
-					paths: []string{filepath.Join(testdataDir, "pluginRootWithDist", "datasource")},
-					class: plugins.ClassExternal,
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "datasource")},
+					strictMode: true,
+					class:      plugins.ClassExternal,
 				},
 				{
-					paths: []string{filepath.Join(testdataDir, "pluginRootWithDist", "dist")},
-					class: plugins.ClassExternal,
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "dist")},
+					strictMode: true,
+					class:      plugins.ClassExternal,
 				},
 				{
-					paths: []string{filepath.Join(testdataDir, "pluginRootWithDist", "panel")},
-					class: plugins.ClassExternal,
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "panel")},
+					strictMode: true,
+					class:      plugins.ClassExternal,
+				},
+			},
+		},
+		{
+			name: "Dev mode disables strict mode for source",
+			cfg: &config.PluginManagementCfg{
+				DevMode: true,
+			},
+			pluginsPath: filepath.Join(testdataDir, "pluginRootWithDist"),
+			expected: []*LocalSource{
+				{
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "datasource")},
+					class:      plugins.ClassExternal,
+					strictMode: false,
+				},
+				{
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "dist")},
+					class:      plugins.ClassExternal,
+					strictMode: false,
+				},
+				{
+					paths:      []string{filepath.Join(testdataDir, "pluginRootWithDist", "panel")},
+					class:      plugins.ClassExternal,
+					strictMode: false,
 				},
 			},
 		},
 		{
 			name:        "Directory with no subdirectories",
+			cfg:         &config.PluginManagementCfg{},
 			pluginsPath: filepath.Join(testdataDir, "pluginRootWithDist", "datasource"),
 			expected:    []*LocalSource{},
 		},
 		{
 			name:        "Directory with a symlink to a directory",
 			pluginsPath: filepath.Join(testdataDir, "symbolic-plugin-dirs"),
+			cfg:         &config.PluginManagementCfg{},
 			expected: []*LocalSource{
 				{
-					paths: []string{filepath.Join(testdataDir, "symbolic-plugin-dirs", "plugin")},
-					class: plugins.ClassExternal,
+					paths:      []string{filepath.Join(testdataDir, "symbolic-plugin-dirs", "plugin")},
+					class:      plugins.ClassExternal,
+					strictMode: true,
 				},
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := DirAsLocalSources(tt.pluginsPath, plugins.ClassExternal)
+			got, err := DirAsLocalSources(tt.cfg, tt.pluginsPath, plugins.ClassExternal)
 			if tt.err != nil {
 				require.Errorf(t, err, tt.err.Error())
+				return
+			}
+			require.NoError(t, err)
+			if !cmp.Equal(got, tt.expected, compareOpts...) {
+				t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.expected, compareOpts...))
 			}
-			require.Equal(t, tt.expected, got)
 		})
 	}
 }

pkg/plugins/manager/sources/sources.go

@@ -5,25 +5,32 @@ import (
 	"path/filepath"
 
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
 	"github.com/grafana/grafana/pkg/plugins/log"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
 type Service struct {
-	cfg *setting.Cfg
+	cfg            *config.PluginManagementCfg
+	staticRootPath string
+
 	log log.Logger
 }
 
-func ProvideService(cfg *setting.Cfg) *Service {
+func ProvideService(cfg *setting.Cfg, pCcfg *config.PluginManagementCfg) *Service {
 	return &Service{
-		cfg: cfg,
-		log: log.New("plugin.sources"),
+		cfg:            pCcfg,
+		staticRootPath: cfg.StaticRootPath,
+		log:            log.New("plugin.sources"),
 	}
 }
 
 func (s *Service) List(_ context.Context) []plugins.PluginSource {
 	r := []plugins.PluginSource{
-		NewLocalSource(plugins.ClassCore, corePluginPaths(s.cfg.StaticRootPath)),
+		NewLocalSource(
+			plugins.ClassCore,
+			s.corePluginPaths(),
+		),
 	}
 	r = append(r, s.externalPluginSources()...)
 	r = append(r, s.pluginSettingSources()...)
@@ -31,7 +38,7 @@ func (s *Service) List(_ context.Context) []plugins.PluginSource {
 }
 
 func (s *Service) externalPluginSources() []plugins.PluginSource {
-	localSrcs, err := DirAsLocalSources(s.cfg.PluginsPath, plugins.ClassExternal)
+	localSrcs, err := DirAsLocalSources(s.cfg, s.cfg.PluginsPath, plugins.ClassExternal)
 	if err != nil {
 		s.log.Error("Failed to load external plugins", "error", err)
 		return []plugins.PluginSource{}
@@ -52,16 +59,19 @@ func (s *Service) pluginSettingSources() []plugins.PluginSource {
 		if !exists || path == "" {
 			continue
 		}
-
-		sources = append(sources, NewLocalSource(plugins.ClassExternal, []string{path}))
+		if s.cfg.DevMode {
+			sources = append(sources, NewUnsafeLocalSource(plugins.ClassExternal, []string{path}))
+		} else {
+			sources = append(sources, NewLocalSource(plugins.ClassExternal, []string{path}))
+		}
 	}
 
 	return sources
 }
 
 // corePluginPaths provides a list of the Core plugin file system paths
-func corePluginPaths(staticRootPath string) []string {
-	datasourcePaths := filepath.Join(staticRootPath, "app/plugins/datasource")
-	panelsPath := filepath.Join(staticRootPath, "app/plugins/panel")
+func (s *Service) corePluginPaths() []string {
+	datasourcePaths := filepath.Join(s.staticRootPath, "app", "plugins", "datasource")
+	panelsPath := filepath.Join(s.staticRootPath, "app", "plugins", "panel")
 	return []string{datasourcePaths, panelsPath}
 }

pkg/plugins/manager/sources/sources_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -18,7 +19,10 @@ func TestSources_List(t *testing.T) {
 
 		cfg := &setting.Cfg{
 			StaticRootPath: testdata,
-			PluginsPath:    filepath.Join(testdata, "pluginRootWithDist"),
+		}
+
+		pCfg := &config.PluginManagementCfg{
+			PluginsPath: filepath.Join(testdata, "pluginRootWithDist"),
 			PluginSettings: setting.PluginSettings{
 				"foo": map[string]string{
 					"path": filepath.Join(testdata, "test-app"),
@@ -29,7 +33,7 @@ func TestSources_List(t *testing.T) {
 			},
 		}
 
-		s := ProvideService(cfg)
+		s := ProvideService(cfg, pCfg)
 		srcs := s.List(context.Background())
 
 		ctx := context.Background()
@@ -37,10 +41,14 @@ func TestSources_List(t *testing.T) {
 		require.Len(t, srcs, 5)
 
 		require.Equal(t, srcs[0].PluginClass(ctx), plugins.ClassCore)
-		require.Equal(t, srcs[0].PluginURIs(ctx), []string{
-			filepath.Join(testdata, "app", "plugins", "datasource"),
-			filepath.Join(testdata, "app", "plugins", "panel"),
-		})
+		if localSrc, ok := srcs[0].(*LocalSource); ok {
+			require.Equal(t, localSrc.Paths(), []string{
+				filepath.Join(testdata, "app", "plugins", "datasource"),
+				filepath.Join(testdata, "app", "plugins", "panel"),
+			})
+		} else {
+			t.Fatalf("Expected LocalSource, got %T", srcs[0])
+		}
 		sig, exists := srcs[0].DefaultSignature(ctx, "")
 		require.True(t, exists)
 		require.Equal(t, plugins.SignatureStatusInternal, sig.Status)
@@ -48,25 +56,37 @@ func TestSources_List(t *testing.T) {
 		require.Equal(t, "", sig.SigningOrg)
 
 		require.Equal(t, srcs[1].PluginClass(ctx), plugins.ClassExternal)
-		require.Equal(t, srcs[1].PluginURIs(ctx), []string{
-			filepath.Join(testdata, "pluginRootWithDist", "datasource"),
-		})
+		if localSrc, ok := srcs[1].(*LocalSource); ok {
+			require.Equal(t, localSrc.Paths(), []string{
+				filepath.Join(testdata, "pluginRootWithDist", "datasource"),
+			})
+		} else {
+			t.Fatalf("Expected LocalSource, got %T", srcs[1])
+		}
 		sig, exists = srcs[1].DefaultSignature(ctx, "")
 		require.False(t, exists)
 		require.Equal(t, plugins.Signature{}, sig)
 
 		require.Equal(t, srcs[2].PluginClass(ctx), plugins.ClassExternal)
-		require.Equal(t, srcs[2].PluginURIs(ctx), []string{
-			filepath.Join(testdata, "pluginRootWithDist", "dist"),
-		})
+		if localSrc, ok := srcs[2].(*LocalSource); ok {
+			require.Equal(t, localSrc.Paths(), []string{
+				filepath.Join(testdata, "pluginRootWithDist", "dist"),
+			})
+		} else {
+			t.Fatalf("Expected LocalSource, got %T", srcs[2])
+		}
 		sig, exists = srcs[2].DefaultSignature(ctx, "")
 		require.False(t, exists)
 		require.Equal(t, plugins.Signature{}, sig)
 
 		require.Equal(t, srcs[3].PluginClass(ctx), plugins.ClassExternal)
-		require.Equal(t, srcs[3].PluginURIs(ctx), []string{
-			filepath.Join(testdata, "pluginRootWithDist", "panel"),
-		})
+		if localSrc, ok := srcs[3].(*LocalSource); ok {
+			require.Equal(t, localSrc.Paths(), []string{
+				filepath.Join(testdata, "pluginRootWithDist", "panel"),
+			})
+		} else {
+			t.Fatalf("Expected LocalSource, got %T", srcs[3])
+		}
 		sig, exists = srcs[3].DefaultSignature(ctx, "")
 		require.False(t, exists)
 		require.Equal(t, plugins.Signature{}, sig)
@@ -78,19 +98,25 @@ func TestSources_List(t *testing.T) {
 
 		cfg := &setting.Cfg{
 			StaticRootPath: testdata,
-			PluginsPath:    filepath.Join(testdata, "symbolic-plugin-dirs"),
 		}
-		s := ProvideService(cfg)
+
+		pCfg := &config.PluginManagementCfg{
+			PluginsPath: filepath.Join(testdata, "symbolic-plugin-dirs"),
+		}
+
+		s := ProvideService(cfg, pCfg)
 		ctx := context.Background()
 		srcs := s.List(ctx)
 		uris := map[plugins.Class]map[string]struct{}{}
-		for _, s := range srcs {
-			class := s.PluginClass(ctx)
+		for _, src := range srcs {
+			class := src.PluginClass(ctx)
 			if _, exists := uris[class]; !exists {
 				uris[class] = map[string]struct{}{}
 			}
-			for _, uri := range s.PluginURIs(ctx) {
-				uris[class][uri] = struct{}{}
+			if localSrc, ok := src.(*LocalSource); ok {
+				for _, path := range localSrc.Paths() {
+					uris[class][path] = struct{}{}
+				}
 			}
 		}