Plugins: Plugins loader pipeline (#71438)

* discovery

* flesh out

* add docs

* remove unused func

* bootstrap stage

* fix docs

* update docs

* undo unnecessary changes

* add end tag

* update doc

* fix linter

* fix

* tidy

* update docs

* add class to filter func

* apply PR feedback

* fix test

pkg/plugins/manager/pipeline/discovery/discovery.go

@@ -0,0 +1,74 @@
+package discovery
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+)
+
+// Discoverer is responsible for the Discovery stage of the plugin loader pipeline.
+type Discoverer interface {
+	Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+}
+
+// FindFunc is the function used for the Find step of the Discovery stage.
+type FindFunc func(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+
+// FindFilterFunc is the function used for the Filter step of the Discovery stage.
+type FindFilterFunc func(ctx context.Context, class plugins.Class, bundles []*plugins.FoundBundle) ([]*plugins.FoundBundle, error)
+
+// Discovery implements the Discoverer interface.
+//
+// The Discovery stage is made up of the following steps (in order):
+// - Find: Find plugins (from disk, remote, etc.)
+// - Filter: Filter the results based on some criteria.
+//
+// The Find step is implemented by the FindFunc type.
+//
+// The Filter step is implemented by the FindFilterFunc type.
+type Discovery struct {
+	findStep        FindFunc
+	findFilterSteps []FindFilterFunc
+	log             log.Logger
+}
+
+type Opts struct {
+	FindFunc        FindFunc
+	FindFilterFuncs []FindFilterFunc
+}
+
+// New returns a new Discovery stage.
+func New(cfg *config.Cfg, opts Opts) *Discovery {
+	if opts.FindFunc == nil {
+		opts.FindFunc = DefaultFindFunc(cfg)
+	}
+
+	if len(opts.FindFilterFuncs) == 0 {
+		opts.FindFilterFuncs = []FindFilterFunc{} // no filters by default
+	}
+
+	return &Discovery{
+		findStep:        opts.FindFunc,
+		findFilterSteps: opts.FindFilterFuncs,
+		log:             log.New("plugins.discovery"),
+	}
+}
+
+// Discover will execute the Find and Filter steps of the Discovery stage.
+func (d *Discovery) Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error) {
+	found, err := d.findStep(ctx, src)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, filterStep := range d.findFilterSteps {
+		found, err = filterStep(ctx, src.PluginClass(ctx), found)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return found, nil
+}

pkg/plugins/manager/pipeline/discovery/doc.go

@@ -0,0 +1,6 @@
+// Package discovery defines the first stage of the plugin loader pipeline.
+
+// The Discovery stage must implement the Discoverer interface.
+// - Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+
+package discovery

pkg/plugins/manager/pipeline/discovery/steps.go

@@ -0,0 +1,55 @@
+package discovery
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/loader/finder"
+	"github.com/grafana/grafana/pkg/plugins/manager/registry"
+)
+
+// DefaultFindFunc is the default function used for the Find step of the Discovery stage. It will scan the local
+// filesystem for plugins.
+func DefaultFindFunc(cfg *config.Cfg) FindFunc {
+	return finder.NewLocalFinder(cfg.DevMode).Find
+}
+
+// DuplicatePluginValidation is a filter step that will filter out any plugins that are already registered with the
+// registry. This includes both the primary plugin and any child plugins, which are matched using the plugin ID field.
+type DuplicatePluginValidation struct {
+	registry registry.Service
+	log      log.Logger
+}
+
+// NewDuplicatePluginFilterStep returns a new DuplicatePluginValidation.
+func NewDuplicatePluginFilterStep(registry registry.Service) *DuplicatePluginValidation {
+	return &DuplicatePluginValidation{
+		registry: registry,
+		log:      log.New("plugins.dedupe"),
+	}
+}
+
+// Filter will filter out any plugins that are already registered with the registry.
+func (d *DuplicatePluginValidation) Filter(ctx context.Context, bundles []*plugins.FoundBundle) ([]*plugins.FoundBundle, error) {
+	res := make([]*plugins.FoundBundle, 0, len(bundles))
+	for _, b := range bundles {
+		_, exists := d.registry.Plugin(ctx, b.Primary.JSONData.ID)
+		if exists {
+			d.log.Warn("Skipping loading of plugin as it's a duplicate", "pluginID", b.Primary.JSONData.ID)
+			continue
+		}
+
+		for _, child := range b.Children {
+			_, exists = d.registry.Plugin(ctx, child.JSONData.ID)
+			if exists {
+				d.log.Warn("Skipping loading of child plugin as it's a duplicate", "pluginID", child.JSONData.ID)
+				continue
+			}
+		}
+		res = append(res, b)
+	}
+
+	return res, nil
+}