Plugins: Refactor Plugin Management (#40477)

* add core plugin flow

* add instrumentation

* move func

* remove cruft

* support external backend plugins

* refactor + clean up

* remove comments

* refactor loader

* simplify core plugin path arg

* cleanup loggers

* move signature validator to plugins package

* fix sig packaging

* cleanup plugin model

* remove unnecessary plugin field

* add start+stop for pm

* fix failures

* add decommissioned state

* export fields just to get things flowing

* fix comments

* set static routes

* make image loading idempotent

* merge with backend plugin manager

* re-use funcs

* reorder imports + remove unnecessary interface

* add some TODOs + remove unused func

* remove unused instrumentation func

* simplify client usage

* remove import alias

* re-use backendplugin.Plugin interface

* re order funcs

* improve var name

* fix log statements

* refactor data model

* add logic for dupe check during loading

* cleanup state setting

* refactor loader

* cleanup manager interface

* add rendering flow

* refactor loading + init

* add renderer support

* fix renderer plugin

* reformat imports

* track errors

* fix plugin signature inheritance

* name param in interface

* update func comment

* fix func arg name

* introduce class concept

* remove func

* fix external plugin check

* apply changes from pm-experiment

* fix core plugins

* fix imports

* rename interface

* comment API interface

* add support for testdata plugin

* enable alerting + use correct core plugin contracts

* slim manager API

* fix param name

* fix filter

* support static routes

* fix rendering

* tidy rendering

* get tests compiling

* fix install+uninstall

* start finder test

* add finder test coverage

* start loader tests

* add test for core plugins

* load core + bundled test

* add test for nested plugin loading

* add test files

* clean interface + fix registering some core plugins

* refactoring

* reformat and create sub packages

* simplify core plugin init

* fix ctx cancel scenario

* migrate initializer

* remove Init() funcs

* add test starter

* new logger

* flesh out initializer tests

* refactoring

* remove unused svc

* refactor rendering flow

* fixup loader tests

* add enabled helper func

* fix logger name

* fix data fetchers

* fix case where plugin dir doesn't exist

* improve coverage + move dupe checking to loader

* remove noisy debug logs

* register core plugins automagically

* add support for renderer in catalog

* make private func + fix req validation

* use interface

* re-add check for renderer in catalog

* tidy up from moving to auto reg core plugins

* core plugin registrar

* guards

* copy over core plugins for test infra

* all tests green

* renames

* propagate new interfaces

* kill old manager

* get compiling

* tidy up

* update naming

* refactor manager test + cleanup

* add more cases to finder test

* migrate validator to field

* more coverage

* refactor dupe checking

* add test for plugin class

* add coverage for initializer

* split out rendering

* move

* fixup tests

* fix uss test

* fix frontend settings

* fix grafanads test

* add check when checking sig errors

* fix enabled map

* fixup

* allow manual setup of CM

* rename to cloud-monitoring

* remove TODO

* add installer interface for testing

* loader interface returns

* tests passing

* refactor + add more coverage

* support 'stackdriver'

* fix frontend settings loading

* improve naming based on package name

* small tidy

* refactor test

* fix renderer start

* make cloud-monitoring plugin ID clearer

* add plugin update test

* add integration tests

* don't break all if sig can't be calculated

* add root URL check test

* add more signature verification tests

* update DTO name

* update enabled plugins comment

* update comments

* fix linter

* revert fe naming change

* fix errors endpoint

* reset error code field name

* re-order test to help verify

* assert -> require

* pm check

* add missing entry + re-order

* re-check

* dump icon log

* verify manager contents first

* reformat

* apply PR feedback

* apply style changes

* fix one vs all loading err

* improve log output

* only start when no signature error

* move log

* rework plugin update check

* fix test

* fix multi loading from cfg.PluginSettings

* improve log output #2

* add error abstraction to capture errors without registering a plugin

* add debug log

* add unsigned warning

* e2e test attempt

* fix logger

* set home path

* prevent panic

* alternate

* ugh.. fix home path

* return renderer even if not started

* make renderer plugin managed

* add fallback renderer icon, update renderer badge + prevent changes when renderer is installed

* fix icon loading

* rollback renderer changes

* use correct field

* remove unneccessary block

* remove newline

* remove unused func

* fix bundled plugins base + module fields

* remove unused field since refactor

* add authorizer abstraction

* loader only returns plugins expected to run

* fix multi log output

pkg/api/frontendsettings.go

@@ -15,7 +15,7 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )
 
-func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins *plugins.EnabledPlugins) (map[string]interface{}, error) {
+func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins EnabledPlugins) (map[string]interface{}, error) {
 	orgDataSources := make([]*models.DataSource, 0)
 
 	if c.OrgId != 0 {
@@ -61,12 +61,19 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins *plu
 			"access":    ds.Access,
 		}
 
-		meta, exists := enabledPlugins.DataSources[ds.Type]
+		meta, exists := enabledPlugins.Get(plugins.DataSource, ds.Type)
 		if !exists {
 			log.Error("Could not find plugin definition for data source", "datasource_type", ds.Type)
 			continue
 		}
-		dsMap["meta"] = meta
+		dsMap["preload"] = meta.Preload
+		dsMap["module"] = meta.Module
+		dsMap["meta"] = &plugins.PluginMetaDTO{
+			JSONData:  meta.JSONData,
+			Signature: meta.Signature,
+			Module:    meta.Module,
+			BaseURL:   meta.BaseURL,
+		}
 
 		jsonData := ds.JsonData
 		if jsonData == nil {
@@ -113,12 +120,17 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins *plu
 
 	// add data sources that are built in (meaning they are not added via data sources page, nor have any entry in
 	// the datasource table)
-	for _, ds := range hs.PluginManager.DataSources() {
+	for _, ds := range hs.pluginStore.Plugins(plugins.DataSource) {
 		if ds.BuiltIn {
 			info := map[string]interface{}{
 				"type": ds.Type,
 				"name": ds.Name,
-				"meta": hs.PluginManager.GetDataSource(ds.Id),
+				"meta": &plugins.PluginMetaDTO{
+					JSONData:  ds.JSONData,
+					Signature: ds.Signature,
+					Module:    ds.Module,
+					BaseURL:   ds.BaseURL,
+				},
 			}
 			if ds.Name == grafanads.DatasourceName {
 				info["id"] = grafanads.DatasourceID
@@ -133,13 +145,13 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins *plu
 
 // getFrontendSettingsMap returns a json object with all the settings needed for front end initialisation.
 func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]interface{}, error) {
-	enabledPlugins, err := hs.PluginManager.GetEnabledPlugins(c.OrgId)
+	enabledPlugins, err := hs.enabledPlugins(c.OrgId)
 	if err != nil {
 		return nil, err
 	}
 
 	pluginsToPreload := []string{}
-	for _, app := range enabledPlugins.Apps {
+	for _, app := range enabledPlugins[plugins.App] {
 		if app.Preload {
 			pluginsToPreload = append(pluginsToPreload, app.Module)
 		}
@@ -157,15 +169,15 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 			defaultDS = n
 		}
 
-		meta := dsM["meta"].(*plugins.DataSourcePlugin)
-		if meta.Preload {
-			pluginsToPreload = append(pluginsToPreload, meta.Module)
+		module, _ := dsM["module"].(string)
+		if preload, _ := dsM["preload"].(bool); preload && module != "" {
+			pluginsToPreload = append(pluginsToPreload, module)
 		}
 	}
 
 	panels := map[string]interface{}{}
-	for _, panel := range enabledPlugins.Panels {
-		if panel.State == plugins.PluginStateAlpha && !hs.Cfg.PluginsEnableAlpha {
+	for _, panel := range enabledPlugins[plugins.Panel] {
+		if panel.State == plugins.AlphaRelease && !hs.Cfg.PluginsEnableAlpha {
 			continue
 		}
 
@@ -173,14 +185,14 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 			pluginsToPreload = append(pluginsToPreload, panel.Module)
 		}
 
-		panels[panel.Id] = map[string]interface{}{
+		panels[panel.ID] = map[string]interface{}{
+			"id":            panel.ID,
 			"module":        panel.Module,
-			"baseUrl":       panel.BaseUrl,
+			"baseUrl":       panel.BaseURL,
 			"name":          panel.Name,
-			"id":            panel.Id,
 			"info":          panel.Info,
 			"hideFromList":  panel.HideFromList,
-			"sort":          getPanelSort(panel.Id),
+			"sort":          getPanelSort(panel.ID),
 			"skipDataQuery": panel.SkipDataQuery,
 			"state":         panel.State,
 			"signature":     panel.Signature,
@@ -241,8 +253,8 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 			"commit":        commit,
 			"buildstamp":    buildstamp,
 			"edition":       hs.License.Edition(),
-			"latestVersion": hs.PluginManager.GrafanaLatestVersion(),
-			"hasUpdate":     hs.PluginManager.GrafanaHasUpdate(),
+			"latestVersion": hs.updateChecker.LatestGrafanaVersion(),
+			"hasUpdate":     hs.updateChecker.GrafanaUpdateAvailable(),
 			"env":           setting.Env,
 			"isEnterprise":  hs.License.HasValidLicense(),
 		},
@@ -335,3 +347,96 @@ func (hs *HTTPServer) GetFrontendSettings(c *models.ReqContext) {
 
 	c.JSON(200, settings)
 }
+
+// EnabledPlugins represents a mapping from plugin types (panel, data source, etc.) to plugin IDs to plugins
+// For example ["panel"] -> ["piechart"] -> {pie chart plugin instance}
+type EnabledPlugins map[plugins.Type]map[string]*plugins.Plugin
+
+func (ep EnabledPlugins) Get(pluginType plugins.Type, pluginID string) (*plugins.Plugin, bool) {
+	if _, exists := ep[pluginType][pluginID]; exists {
+		return ep[pluginType][pluginID], true
+	}
+
+	return nil, false
+}
+
+func (hs *HTTPServer) enabledPlugins(orgID int64) (EnabledPlugins, error) {
+	ep := make(EnabledPlugins)
+
+	pluginSettingMap, err := hs.pluginSettings(orgID)
+	if err != nil {
+		return ep, err
+	}
+
+	apps := make(map[string]*plugins.Plugin)
+	for _, app := range hs.pluginStore.Plugins(plugins.App) {
+		if b, exists := pluginSettingMap[app.ID]; exists {
+			app.Pinned = b.Pinned
+			apps[app.ID] = app
+		}
+	}
+	ep[plugins.App] = apps
+
+	dataSources := make(map[string]*plugins.Plugin)
+	for _, ds := range hs.pluginStore.Plugins(plugins.DataSource) {
+		if _, exists := pluginSettingMap[ds.ID]; exists {
+			dataSources[ds.ID] = ds
+		}
+	}
+	ep[plugins.DataSource] = dataSources
+
+	panels := make(map[string]*plugins.Plugin)
+	for _, p := range hs.pluginStore.Plugins(plugins.Panel) {
+		if _, exists := pluginSettingMap[p.ID]; exists {
+			panels[p.ID] = p
+		}
+	}
+	ep[plugins.Panel] = panels
+
+	return ep, nil
+}
+
+func (hs *HTTPServer) pluginSettings(orgID int64) (map[string]*models.PluginSettingInfoDTO, error) {
+	pluginSettings, err := hs.SQLStore.GetPluginSettings(orgID)
+	if err != nil {
+		return nil, err
+	}
+
+	pluginMap := make(map[string]*models.PluginSettingInfoDTO)
+	for _, plug := range pluginSettings {
+		pluginMap[plug.PluginId] = plug
+	}
+
+	for _, pluginDef := range hs.pluginStore.Plugins() {
+		// ignore entries that already exist
+		if _, ok := pluginMap[pluginDef.ID]; ok {
+			continue
+		}
+
+		// enabled by default
+		opt := &models.PluginSettingInfoDTO{
+			PluginId: pluginDef.ID,
+			OrgId:    orgID,
+			Enabled:  true,
+		}
+
+		// apps are disabled by default unless autoEnabled: true
+		if p := hs.pluginStore.Plugin(pluginDef.ID); p != nil && p.IsApp() {
+			opt.Enabled = p.AutoEnabled
+			opt.Pinned = p.AutoEnabled
+		}
+
+		// if it's included in app, check app settings
+		if pluginDef.IncludedInAppID != "" {
+			// app components are by default disabled
+			opt.Enabled = false
+
+			if appSettings, ok := pluginMap[pluginDef.IncludedInAppID]; ok {
+				opt.Enabled = appSettings.Enabled
+			}
+		}
+		pluginMap[pluginDef.ID] = opt
+	}
+
+	return pluginMap, nil
+}