Plugins: Refactor Plugin Management (#40477)

* add core plugin flow

* add instrumentation

* move func

* remove cruft

* support external backend plugins

* refactor + clean up

* remove comments

* refactor loader

* simplify core plugin path arg

* cleanup loggers

* move signature validator to plugins package

* fix sig packaging

* cleanup plugin model

* remove unnecessary plugin field

* add start+stop for pm

* fix failures

* add decommissioned state

* export fields just to get things flowing

* fix comments

* set static routes

* make image loading idempotent

* merge with backend plugin manager

* re-use funcs

* reorder imports + remove unnecessary interface

* add some TODOs + remove unused func

* remove unused instrumentation func

* simplify client usage

* remove import alias

* re-use backendplugin.Plugin interface

* re order funcs

* improve var name

* fix log statements

* refactor data model

* add logic for dupe check during loading

* cleanup state setting

* refactor loader

* cleanup manager interface

* add rendering flow

* refactor loading + init

* add renderer support

* fix renderer plugin

* reformat imports

* track errors

* fix plugin signature inheritance

* name param in interface

* update func comment

* fix func arg name

* introduce class concept

* remove func

* fix external plugin check

* apply changes from pm-experiment

* fix core plugins

* fix imports

* rename interface

* comment API interface

* add support for testdata plugin

* enable alerting + use correct core plugin contracts

* slim manager API

* fix param name

* fix filter

* support static routes

* fix rendering

* tidy rendering

* get tests compiling

* fix install+uninstall

* start finder test

* add finder test coverage

* start loader tests

* add test for core plugins

* load core + bundled test

* add test for nested plugin loading

* add test files

* clean interface + fix registering some core plugins

* refactoring

* reformat and create sub packages

* simplify core plugin init

* fix ctx cancel scenario

* migrate initializer

* remove Init() funcs

* add test starter

* new logger

* flesh out initializer tests

* refactoring

* remove unused svc

* refactor rendering flow

* fixup loader tests

* add enabled helper func

* fix logger name

* fix data fetchers

* fix case where plugin dir doesn't exist

* improve coverage + move dupe checking to loader

* remove noisy debug logs

* register core plugins automagically

* add support for renderer in catalog

* make private func + fix req validation

* use interface

* re-add check for renderer in catalog

* tidy up from moving to auto reg core plugins

* core plugin registrar

* guards

* copy over core plugins for test infra

* all tests green

* renames

* propagate new interfaces

* kill old manager

* get compiling

* tidy up

* update naming

* refactor manager test + cleanup

* add more cases to finder test

* migrate validator to field

* more coverage

* refactor dupe checking

* add test for plugin class

* add coverage for initializer

* split out rendering

* move

* fixup tests

* fix uss test

* fix frontend settings

* fix grafanads test

* add check when checking sig errors

* fix enabled map

* fixup

* allow manual setup of CM

* rename to cloud-monitoring

* remove TODO

* add installer interface for testing

* loader interface returns

* tests passing

* refactor + add more coverage

* support 'stackdriver'

* fix frontend settings loading

* improve naming based on package name

* small tidy

* refactor test

* fix renderer start

* make cloud-monitoring plugin ID clearer

* add plugin update test

* add integration tests

* don't break all if sig can't be calculated

* add root URL check test

* add more signature verification tests

* update DTO name

* update enabled plugins comment

* update comments

* fix linter

* revert fe naming change

* fix errors endpoint

* reset error code field name

* re-order test to help verify

* assert -> require

* pm check

* add missing entry + re-order

* re-check

* dump icon log

* verify manager contents first

* reformat

* apply PR feedback

* apply style changes

* fix one vs all loading err

* improve log output

* only start when no signature error

* move log

* rework plugin update check

* fix test

* fix multi loading from cfg.PluginSettings

* improve log output #2

* add error abstraction to capture errors without registering a plugin

* add debug log

* add unsigned warning

* e2e test attempt

* fix logger

* set home path

* prevent panic

* alternate

* ugh.. fix home path

* return renderer even if not started

* make renderer plugin managed

* add fallback renderer icon, update renderer badge + prevent changes when renderer is installed

* fix icon loading

* rollback renderer changes

* use correct field

* remove unneccessary block

* remove newline

* remove unused func

* fix bundled plugins base + module fields

* remove unused field since refactor

* add authorizer abstraction

* loader only returns plugins expected to run

* fix multi log output

pkg/plugins/ifaces.go

@@ -3,60 +3,102 @@ package plugins
 import (
 	"context"
 
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/plugins/backendplugin"
 )
 
-// Manager is the plugin manager service interface.
-type Manager interface {
-	// Renderer gets the renderer plugin.
-	Renderer() *RendererPlugin
-	// GetDataSource gets a data source plugin with a certain ID.
-	GetDataSource(id string) *DataSourcePlugin
-	// GetPlugin gets a plugin with a certain ID.
-	GetPlugin(id string) *PluginBase
-	// GetApp gets an app plugin with a certain ID.
-	GetApp(id string) *AppPlugin
-	// DataSourceCount gets the number of data sources.
-	DataSourceCount() int
-	// DataSources gets all data sources.
-	DataSources() []*DataSourcePlugin
-	// Apps gets all app plugins.
-	Apps() []*AppPlugin
-	// PanelCount gets the number of panels.
-	PanelCount() int
-	// AppCount gets the number of apps.
-	AppCount() int
-	// GetEnabledPlugins gets enabled plugins.
-	GetEnabledPlugins(orgID int64) (*EnabledPlugins, error)
-	// GrafanaLatestVersion gets the latest Grafana version.
-	GrafanaLatestVersion() string
-	// GrafanaHasUpdate returns whether Grafana has an update.
-	GrafanaHasUpdate() bool
-	// Plugins gets all plugins.
-	Plugins() []*PluginBase
-	// StaticRoutes gets all static routes.
-	StaticRoutes() []*PluginStaticRoute
-	// GetPluginSettings gets settings for a certain plugin.
-	GetPluginSettings(orgID int64) (map[string]*models.PluginSettingInfoDTO, error)
+// DataRequestHandler is a data request handler interface.
+type DataRequestHandler interface {
+	// HandleRequest handles a data request.
+	HandleRequest(context.Context, *models.DataSource, DataQuery) (DataResponse, error)
+}
+
+// Store is the storage for plugins.
+type Store interface {
+	// Plugin finds a plugin by its ID.
+	Plugin(pluginID string) *Plugin
+	// Plugins returns plugins by their requested type.
+	Plugins(pluginTypes ...Type) []*Plugin
+
+	// Add adds a plugin to the store.
+	Add(ctx context.Context, pluginID, version string, opts AddOpts) error
+	// Remove removes a plugin from the store.
+	Remove(ctx context.Context, pluginID string) error
+}
+
+type AddOpts struct {
+	PluginInstallDir, PluginZipURL, PluginRepoURL string
+}
+
+// Loader is responsible for loading plugins from the file system.
+type Loader interface {
+	// Load will return a list of plugins found in the provided file system paths.
+	Load(paths []string, ignore map[string]struct{}) ([]*Plugin, error)
+	// LoadWithFactory will return a plugin found in the provided file system path and use the provided factory to
+	// construct the plugin backend client.
+	LoadWithFactory(path string, factory backendplugin.PluginFactoryFunc) (*Plugin, error)
+}
+
+// Installer is responsible for managing plugins (add / remove) on the file system.
+type Installer interface {
+	// Install downloads the requested plugin in the provided file system location.
+	Install(ctx context.Context, pluginID, version, pluginsDir, pluginZipURL, pluginRepoURL string) error
+	// Uninstall removes the requested plugin from the provided file system location.
+	Uninstall(ctx context.Context, pluginDir string) error
+	// GetUpdateInfo provides update information for the requested plugin.
+	GetUpdateInfo(ctx context.Context, pluginID, version, pluginRepoURL string) (UpdateInfo, error)
+}
+
+type UpdateInfo struct {
+	PluginZipURL string
+}
+
+// Client is used to communicate with backend plugin implementations.
+type Client interface {
+	backend.QueryDataHandler
+	backend.CheckHealthHandler
+
+	// CallResource calls a plugin resource.
+	CallResource(pCtx backend.PluginContext, ctx *models.ReqContext, path string)
+	// CollectMetrics collects metrics from a plugin.
+	CollectMetrics(ctx context.Context, pluginID string) (*backend.CollectMetricsResult, error)
+}
+
+type RendererManager interface {
+	// Renderer returns a renderer plugin.
+	Renderer() *Plugin
+}
+
+type CoreBackendRegistrar interface {
+	// LoadAndRegister loads and registers a Core backend plugin
+	LoadAndRegister(pluginID string, factory backendplugin.PluginFactoryFunc) error
+}
+
+type StaticRouteResolver interface {
+	Routes() []*StaticRoute
+}
+
+type ErrorResolver interface {
+	PluginErrors() []*Error
+}
+
+type PluginLoaderAuthorizer interface {
+	// CanLoadPlugin confirms if a plugin is authorized to load
+	CanLoadPlugin(plugin *Plugin) bool
+}
+
+type PluginDashboardManager interface {
 	// GetPluginDashboards gets dashboards for a certain org/plugin.
 	GetPluginDashboards(orgID int64, pluginID string) ([]*PluginDashboardInfoDTO, error)
-	// GetPluginMarkdown gets markdown for a certain plugin/name.
-	GetPluginMarkdown(pluginID string, name string) ([]byte, error)
-	// ImportDashboard imports a dashboard.
-	ImportDashboard(pluginID, path string, orgID, folderID int64, dashboardModel *simplejson.Json,
-		overwrite bool, inputs []ImportDashboardInput, user *models.SignedInUser,
-		requestHandler DataRequestHandler) (PluginDashboardInfoDTO, *models.Dashboard, error)
-	// ScanningErrors returns plugin scanning errors encountered.
-	ScanningErrors() []PluginError
 	// LoadPluginDashboard loads a plugin dashboard.
 	LoadPluginDashboard(pluginID, path string) (*models.Dashboard, error)
-	// IsAppInstalled returns whether an app is installed.
-	IsAppInstalled(id string) bool
-	// Install installs a plugin.
-	Install(ctx context.Context, pluginID, version string) error
-	// Uninstall uninstalls a plugin.
-	Uninstall(ctx context.Context, pluginID string) error
+	// ImportDashboard imports a dashboard.
+	ImportDashboard(pluginID, path string, orgID, folderID int64, dashboardModel *simplejson.Json,
+		overwrite bool, inputs []ImportDashboardInput, user *models.SignedInUser) (PluginDashboardInfoDTO,
+		*models.Dashboard, error)
 }
 
 type ImportDashboardInput struct {
@@ -65,32 +107,3 @@ type ImportDashboardInput struct {
 	Name     string `json:"name"`
 	Value    string `json:"value"`
 }
-
-// DataRequestHandler is a data request handler interface.
-type DataRequestHandler interface {
-	// HandleRequest handles a data request.
-	HandleRequest(context.Context, *models.DataSource, DataQuery) (DataResponse, error)
-}
-
-type PluginInstaller interface {
-	// Install finds the plugin given the provided information and installs in the provided plugins directory.
-	Install(ctx context.Context, pluginID, version, pluginsDirectory, pluginZipURL, pluginRepoURL string) error
-	// Uninstall removes the specified plugin from the provided plugins directory.
-	Uninstall(ctx context.Context, pluginPath string) error
-	// GetUpdateInfo returns update information if the requested plugin is supported on the running system.
-	GetUpdateInfo(pluginID, version, pluginRepoURL string) (UpdateInfo, error)
-}
-
-type PluginInstallerLogger interface {
-	Successf(format string, args ...interface{})
-	Failuref(format string, args ...interface{})
-
-	Info(args ...interface{})
-	Infof(format string, args ...interface{})
-	Debug(args ...interface{})
-	Debugf(format string, args ...interface{})
-	Warn(args ...interface{})
-	Warnf(format string, args ...interface{})
-	Error(args ...interface{})
-	Errorf(format string, args ...interface{})
-}