Plugins: Refactor Plugin Management (#40477)

* add core plugin flow

* add instrumentation

* move func

* remove cruft

* support external backend plugins

* refactor + clean up

* remove comments

* refactor loader

* simplify core plugin path arg

* cleanup loggers

* move signature validator to plugins package

* fix sig packaging

* cleanup plugin model

* remove unnecessary plugin field

* add start+stop for pm

* fix failures

* add decommissioned state

* export fields just to get things flowing

* fix comments

* set static routes

* make image loading idempotent

* merge with backend plugin manager

* re-use funcs

* reorder imports + remove unnecessary interface

* add some TODOs + remove unused func

* remove unused instrumentation func

* simplify client usage

* remove import alias

* re-use backendplugin.Plugin interface

* re order funcs

* improve var name

* fix log statements

* refactor data model

* add logic for dupe check during loading

* cleanup state setting

* refactor loader

* cleanup manager interface

* add rendering flow

* refactor loading + init

* add renderer support

* fix renderer plugin

* reformat imports

* track errors

* fix plugin signature inheritance

* name param in interface

* update func comment

* fix func arg name

* introduce class concept

* remove func

* fix external plugin check

* apply changes from pm-experiment

* fix core plugins

* fix imports

* rename interface

* comment API interface

* add support for testdata plugin

* enable alerting + use correct core plugin contracts

* slim manager API

* fix param name

* fix filter

* support static routes

* fix rendering

* tidy rendering

* get tests compiling

* fix install+uninstall

* start finder test

* add finder test coverage

* start loader tests

* add test for core plugins

* load core + bundled test

* add test for nested plugin loading

* add test files

* clean interface + fix registering some core plugins

* refactoring

* reformat and create sub packages

* simplify core plugin init

* fix ctx cancel scenario

* migrate initializer

* remove Init() funcs

* add test starter

* new logger

* flesh out initializer tests

* refactoring

* remove unused svc

* refactor rendering flow

* fixup loader tests

* add enabled helper func

* fix logger name

* fix data fetchers

* fix case where plugin dir doesn't exist

* improve coverage + move dupe checking to loader

* remove noisy debug logs

* register core plugins automagically

* add support for renderer in catalog

* make private func + fix req validation

* use interface

* re-add check for renderer in catalog

* tidy up from moving to auto reg core plugins

* core plugin registrar

* guards

* copy over core plugins for test infra

* all tests green

* renames

* propagate new interfaces

* kill old manager

* get compiling

* tidy up

* update naming

* refactor manager test + cleanup

* add more cases to finder test

* migrate validator to field

* more coverage

* refactor dupe checking

* add test for plugin class

* add coverage for initializer

* split out rendering

* move

* fixup tests

* fix uss test

* fix frontend settings

* fix grafanads test

* add check when checking sig errors

* fix enabled map

* fixup

* allow manual setup of CM

* rename to cloud-monitoring

* remove TODO

* add installer interface for testing

* loader interface returns

* tests passing

* refactor + add more coverage

* support 'stackdriver'

* fix frontend settings loading

* improve naming based on package name

* small tidy

* refactor test

* fix renderer start

* make cloud-monitoring plugin ID clearer

* add plugin update test

* add integration tests

* don't break all if sig can't be calculated

* add root URL check test

* add more signature verification tests

* update DTO name

* update enabled plugins comment

* update comments

* fix linter

* revert fe naming change

* fix errors endpoint

* reset error code field name

* re-order test to help verify

* assert -> require

* pm check

* add missing entry + re-order

* re-check

* dump icon log

* verify manager contents first

* reformat

* apply PR feedback

* apply style changes

* fix one vs all loading err

* improve log output

* only start when no signature error

* move log

* rework plugin update check

* fix test

* fix multi loading from cfg.PluginSettings

* improve log output #2

* add error abstraction to capture errors without registering a plugin

* add debug log

* add unsigned warning

* e2e test attempt

* fix logger

* set home path

* prevent panic

* alternate

* ugh.. fix home path

* return renderer even if not started

* make renderer plugin managed

* add fallback renderer icon, update renderer badge + prevent changes when renderer is installed

* fix icon loading

* rollback renderer changes

* use correct field

* remove unneccessary block

* remove newline

* remove unused func

* fix bundled plugins base + module fields

* remove unused field since refactor

* add authorizer abstraction

* loader only returns plugins expected to run

* fix multi log output

pkg/services/datasourceproxy/datasourceproxy.go

@@ -18,12 +18,12 @@ import (
 )
 
 func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator models.PluginRequestValidator,
-	pm plugins.Manager, cfg *setting.Cfg, httpClientProvider httpclient.Provider,
+	pluginStore plugins.Store, cfg *setting.Cfg, httpClientProvider httpclient.Provider,
 	oauthTokenService *oauthtoken.Service, dsService *datasources.Service) *DataSourceProxyService {
 	return &DataSourceProxyService{
 		DataSourceCache:        dataSourceCache,
 		PluginRequestValidator: plugReqValidator,
-		PluginManager:          pm,
+		pluginStore:            pluginStore,
 		Cfg:                    cfg,
 		HTTPClientProvider:     httpClientProvider,
 		OAuthTokenService:      oauthTokenService,
@@ -34,7 +34,7 @@ func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator m
 type DataSourceProxyService struct {
 	DataSourceCache        datasources.CacheService
 	PluginRequestValidator models.PluginRequestValidator
-	PluginManager          plugins.Manager
+	pluginStore            plugins.Store
 	Cfg                    *setting.Cfg
 	HTTPClientProvider     httpclient.Provider
 	OAuthTokenService      *oauthtoken.Service
@@ -69,16 +69,15 @@ func (p *DataSourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte
 	}
 
 	// find plugin
-	plugin := p.PluginManager.GetDataSource(ds.Type)
+	plugin := p.pluginStore.Plugin(ds.Type)
 	if plugin == nil {
 		c.JsonApiErr(http.StatusNotFound, "Unable to find datasource plugin", err)
 		return
 	}
 
-	proxy, err := pluginproxy.NewDataSourceProxy(
-		ds, plugin, c, getProxyPath(c), p.Cfg, p.HTTPClientProvider, p.OAuthTokenService, p.DataSourcesService,
-	)
-
+	proxyPath := getProxyPath(c)
+	proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider,
+		p.OAuthTokenService, p.DataSourcesService)
 	if err != nil {
 		if errors.Is(err, datasource.URLValidationError{}) {
 			c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

pkg/services/live/live.go

@@ -26,7 +26,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/plugins/manager"
+	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/plugincontext"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/live/database"
@@ -61,7 +61,7 @@ type CoreGrafanaScope struct {
 }
 
 func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, routeRegister routing.RouteRegister,
-	logsService *cloudwatch.LogsService, pluginManager *manager.PluginManager, cacheService *localcache.CacheService,
+	logsService *cloudwatch.LogsService, pluginStore plugins.Store, cacheService *localcache.CacheService,
 	dataSourceCache datasources.CacheService, sqlStore *sqlstore.SQLStore,
 	usageStatsService usagestats.Service) (*GrafanaLive, error) {
 	g := &GrafanaLive{
@@ -69,7 +69,7 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r
 		PluginContextProvider: plugCtxProvider,
 		RouteRegister:         routeRegister,
 		LogsService:           logsService,
-		PluginManager:         pluginManager,
+		pluginStore:           pluginStore,
 		CacheService:          cacheService,
 		DataSourceCache:       dataSourceCache,
 		SQLStore:              sqlStore,
@@ -361,10 +361,10 @@ type GrafanaLive struct {
 	Cfg                   *setting.Cfg
 	RouteRegister         routing.RouteRegister
 	LogsService           *cloudwatch.LogsService
-	PluginManager         *manager.PluginManager
 	CacheService          *localcache.CacheService
 	DataSourceCache       datasources.CacheService
 	SQLStore              *sqlstore.SQLStore
+	pluginStore           plugins.Store
 
 	node         *centrifuge.Node
 	surveyCaller *survey.Caller
@@ -393,15 +393,14 @@ type GrafanaLive struct {
 }
 
 func (g *GrafanaLive) getStreamPlugin(pluginID string) (backend.StreamHandler, error) {
-	plugin, ok := g.PluginManager.BackendPluginManager.Get(pluginID)
-	if !ok {
+	plugin := g.pluginStore.Plugin(pluginID)
+	if plugin == nil {
 		return nil, fmt.Errorf("plugin not found: %s", pluginID)
 	}
-	streamHandler, ok := plugin.(backend.StreamHandler)
-	if !ok {
-		return nil, fmt.Errorf("%s plugin does not implement StreamHandler: %#v", pluginID, plugin)
+	if plugin.SupportsStreaming() {
+		return plugin, nil
 	}
-	return streamHandler, nil
+	return nil, fmt.Errorf("%s plugin does not implement StreamHandler: %#v", pluginID, plugin)
 }
 
 func (g *GrafanaLive) Run(ctx context.Context) error {

pkg/services/provisioning/plugins/config_reader.go

@@ -17,12 +17,12 @@ type configReader interface {
 }
 
 type configReaderImpl struct {
-	log           log.Logger
-	pluginManager plugins.Manager
+	log         log.Logger
+	pluginStore plugins.Store
 }
 
-func newConfigReader(logger log.Logger, pluginManager plugins.Manager) configReader {
-	return &configReaderImpl{log: logger, pluginManager: pluginManager}
+func newConfigReader(logger log.Logger, pluginStore plugins.Store) configReader {
+	return &configReaderImpl{log: logger, pluginStore: pluginStore}
 }
 
 func (cr *configReaderImpl) readConfig(path string) ([]*pluginsAsConfig, error) {
@@ -113,8 +113,8 @@ func (cr *configReaderImpl) validatePluginsConfig(apps []*pluginsAsConfig) error
 		}
 
 		for _, app := range apps[i].Apps {
-			if !cr.pluginManager.IsAppInstalled(app.PluginID) {
-				return fmt.Errorf("app plugin not installed: %q", app.PluginID)
+			if p := cr.pluginStore.Plugin(app.PluginID); p == nil {
+				return fmt.Errorf("plugin not installed: %q", app.PluginID)
 			}
 		}
 	}

pkg/services/provisioning/plugins/config_reader_test.go

@@ -32,10 +32,10 @@ func TestConfigReader(t *testing.T) {
 	})
 
 	t.Run("Unknown app plugin should return error", func(t *testing.T) {
-		cfgProvider := newConfigReader(log.New("test logger"), fakePluginManager{})
+		cfgProvider := newConfigReader(log.New("test logger"), fakePluginStore{})
 		_, err := cfgProvider.readConfig(unknownApp)
 		require.Error(t, err)
-		require.Equal(t, "app plugin not installed: \"nonexisting\"", err.Error())
+		require.Equal(t, "plugin not installed: \"nonexisting\"", err.Error())
 	})
 
 	t.Run("Read incorrect properties", func(t *testing.T) {
@@ -46,8 +46,8 @@ func TestConfigReader(t *testing.T) {
 	})
 
 	t.Run("Can read correct properties", func(t *testing.T) {
-		pm := fakePluginManager{
-			apps: map[string]*plugins.AppPlugin{
+		pm := fakePluginStore{
+			apps: map[string]*plugins.Plugin{
 				"test-plugin":   {},
 				"test-plugin-2": {},
 			},
@@ -87,13 +87,12 @@ func TestConfigReader(t *testing.T) {
 	})
 }
 
-type fakePluginManager struct {
-	plugins.Manager
+type fakePluginStore struct {
+	plugins.Store
 
-	apps map[string]*plugins.AppPlugin
+	apps map[string]*plugins.Plugin
 }
 
-func (pm fakePluginManager) IsAppInstalled(id string) bool {
-	_, exists := pm.apps[id]
-	return exists
+func (pr fakePluginStore) Plugin(pluginID string) *plugins.Plugin {
+	return pr.apps[pluginID]
 }

pkg/services/provisioning/plugins/plugin_provisioner.go

@@ -11,11 +11,11 @@ import (
 
 // Provision scans a directory for provisioning config files
 // and provisions the app in those files.
-func Provision(configDirectory string, pluginManager plugins.Manager) error {
+func Provision(configDirectory string, pluginStore plugins.Store) error {
 	logger := log.New("provisioning.plugins")
 	ap := PluginProvisioner{
 		log:         logger,
-		cfgProvider: newConfigReader(logger, pluginManager),
+		cfgProvider: newConfigReader(logger, pluginStore),
 	}
 	return ap.applyChanges(configDirectory)
 }

pkg/services/provisioning/provisioning.go

@@ -18,12 +18,12 @@ import (
 	"github.com/grafana/grafana/pkg/util/errutil"
 )
 
-func ProvideService(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, pluginManager plugifaces.Manager,
+func ProvideService(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, pluginStore plugifaces.Store,
 	encryptionService encryption.Service) (*ProvisioningServiceImpl, error) {
 	s := &ProvisioningServiceImpl{
 		Cfg:                     cfg,
 		SQLStore:                sqlStore,
-		PluginManager:           pluginManager,
+		pluginStore:             pluginStore,
 		EncryptionService:       encryptionService,
 		log:                     log.New("provisioning"),
 		newDashboardProvisioner: dashboards.New,
@@ -61,7 +61,7 @@ func newProvisioningServiceImpl(
 	newDashboardProvisioner dashboards.DashboardProvisionerFactory,
 	provisionNotifiers func(string, encryption.Service) error,
 	provisionDatasources func(context.Context, string) error,
-	provisionPlugins func(string, plugifaces.Manager) error,
+	provisionPlugins func(string, plugifaces.Store) error,
 ) *ProvisioningServiceImpl {
 	return &ProvisioningServiceImpl{
 		log:                     log.New("provisioning"),
@@ -75,7 +75,7 @@ func newProvisioningServiceImpl(
 type ProvisioningServiceImpl struct {
 	Cfg                     *setting.Cfg
 	SQLStore                *sqlstore.SQLStore
-	PluginManager           plugifaces.Manager
+	pluginStore             plugifaces.Store
 	EncryptionService       encryption.Service
 	log                     log.Logger
 	pollingCtxCancel        context.CancelFunc
@@ -83,7 +83,7 @@ type ProvisioningServiceImpl struct {
 	dashboardProvisioner    dashboards.DashboardProvisioner
 	provisionNotifiers      func(string, encryption.Service) error
 	provisionDatasources    func(context.Context, string) error
-	provisionPlugins        func(string, plugifaces.Manager) error
+	provisionPlugins        func(string, plugifaces.Store) error
 	mutex                   sync.Mutex
 }
 
@@ -143,7 +143,7 @@ func (ps *ProvisioningServiceImpl) ProvisionDatasources(ctx context.Context) err
 
 func (ps *ProvisioningServiceImpl) ProvisionPlugins() error {
 	appPath := filepath.Join(ps.Cfg.ProvisioningPath, "plugins")
-	err := ps.provisionPlugins(appPath, ps.PluginManager)
+	err := ps.provisionPlugins(appPath, ps.pluginStore)
 	return errutil.Wrap("app provisioning error", err)
 }
 

pkg/services/rendering/plugin_mode.go

@@ -45,7 +45,7 @@ func (rs *RenderingService) renderViaPlugin(ctx context.Context, renderKey strin
 	}
 	rs.log.Debug("Calling renderer plugin", "req", req)
 
-	rsp, err := rs.pluginInfo.GrpcPluginV2.Render(ctx, req)
+	rsp, err := rs.pluginInfo.Renderer.Render(ctx, req)
 	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 		rs.log.Info("Rendering timed out")
 		return nil, ErrTimeout
@@ -88,7 +88,7 @@ func (rs *RenderingService) renderCSVViaPlugin(ctx context.Context, renderKey st
 	}
 	rs.log.Debug("Calling renderer plugin", "req", req)
 
-	rsp, err := rs.pluginInfo.GrpcPluginV2.RenderCSV(ctx, req)
+	rsp, err := rs.pluginInfo.Renderer.RenderCSV(ctx, req)
 	if err != nil {
 		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 			rs.log.Info("Rendering timed out")

pkg/services/rendering/rendering.go

@@ -13,10 +13,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/infra/remotecache"
-
-	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/setting"
@@ -38,19 +37,19 @@ type RenderUser struct {
 
 type RenderingService struct {
 	log             log.Logger
-	pluginInfo      *plugins.RendererPlugin
+	pluginInfo      *plugins.Plugin
 	renderAction    renderFunc
 	renderCSVAction renderCSVFunc
 	domain          string
 	inProgressCount int32
 	version         string
 
-	Cfg                *setting.Cfg
-	RemoteCacheService *remotecache.RemoteCache
-	PluginManager      plugins.Manager
+	Cfg                   *setting.Cfg
+	RemoteCacheService    *remotecache.RemoteCache
+	RendererPluginManager plugins.RendererManager
 }
 
-func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, pm plugins.Manager) (*RenderingService, error) {
+func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm plugins.RendererManager) (*RenderingService, error) {
 	// ensure ImagesDir exists
 	err := os.MkdirAll(cfg.ImagesDir, 0700)
 	if err != nil {
@@ -81,11 +80,11 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, pm p
 	}
 
 	s := &RenderingService{
-		Cfg:                cfg,
-		RemoteCacheService: remoteCache,
-		PluginManager:      pm,
-		log:                log.New("rendering"),
-		domain:             domain,
+		Cfg:                   cfg,
+		RemoteCacheService:    remoteCache,
+		RendererPluginManager: rm,
+		log:                   log.New("rendering"),
+		domain:                domain,
 	}
 	return s, nil
 }
@@ -109,7 +108,7 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 
 	if rs.pluginAvailable() {
 		rs.log = rs.log.New("renderer", "plugin")
-		rs.pluginInfo = rs.PluginManager.Renderer()
+		rs.pluginInfo = rs.RendererPluginManager.Renderer()
 
 		if err := rs.startPlugin(ctx); err != nil {
 			return err
@@ -142,7 +141,7 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 }
 
 func (rs *RenderingService) pluginAvailable() bool {
-	return rs.PluginManager.Renderer() != nil
+	return rs.RendererPluginManager.Renderer() != nil
 }
 
 func (rs *RenderingService) remoteAvailable() bool {
@@ -157,7 +156,7 @@ func (rs *RenderingService) Version() string {
 	return rs.version
 }
 
-func (rs *RenderingService) RenderErrorImage(err error) (*RenderResult, error) {
+func (rs *RenderingService) RenderErrorImage(_ error) (*RenderResult, error) {
 	imgUrl := "public/img/rendering_error.png"
 	imgPath := filepath.Join(setting.HomePath, imgUrl)
 	if _, err := os.Stat(imgPath); errors.Is(err, os.ErrNotExist) {

pkg/services/updatechecker/grafana_update_checker.go

@@ -0,0 +1,120 @@
+package updatechecker
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/hashicorp/go-version"
+)
+
+var (
+	httpClient = http.Client{Timeout: 10 * time.Second}
+	logger     = log.New("update.checker")
+)
+
+type latestJSON struct {
+	Stable  string `json:"stable"`
+	Testing string `json:"testing"`
+}
+
+type Service struct {
+	cfg *setting.Cfg
+
+	hasUpdate     bool
+	latestVersion string
+	mutex         sync.RWMutex
+}
+
+func ProvideService(cfg *setting.Cfg) *Service {
+	s := newUpdateChecker(cfg)
+
+	return s
+}
+
+func newUpdateChecker(cfg *setting.Cfg) *Service {
+	return &Service{
+		cfg: cfg,
+	}
+}
+
+func (s *Service) IsDisabled() bool {
+	return !s.cfg.CheckForUpdates
+}
+
+func (s *Service) Run(ctx context.Context) error {
+	s.checkForUpdates()
+
+	ticker := time.NewTicker(time.Minute * 10)
+	run := true
+
+	for run {
+		select {
+		case <-ticker.C:
+			s.checkForUpdates()
+		case <-ctx.Done():
+			run = false
+		}
+	}
+
+	return ctx.Err()
+}
+
+func (s *Service) checkForUpdates() {
+	resp, err := httpClient.Get("https://raw.githubusercontent.com/grafana/grafana/main/latest.json")
+	if err != nil {
+		logger.Debug("Failed to get latest.json repo from github.com", "error", err)
+		return
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			logger.Warn("Failed to close response body", "err", err)
+		}
+	}()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logger.Debug("Update check failed, reading response from github.com", "error", err)
+		return
+	}
+
+	var latest latestJSON
+	err = json.Unmarshal(body, &latest)
+	if err != nil {
+		logger.Debug("Failed to unmarshal latest.json", "error", err)
+		return
+	}
+
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if strings.Contains(s.cfg.BuildVersion, "-") {
+		s.latestVersion = latest.Testing
+		s.hasUpdate = !strings.HasPrefix(s.cfg.BuildVersion, latest.Testing)
+	} else {
+		s.latestVersion = latest.Stable
+		s.hasUpdate = latest.Stable != s.cfg.BuildVersion
+	}
+
+	currVersion, err1 := version.NewVersion(s.cfg.BuildVersion)
+	latestVersion, err2 := version.NewVersion(s.latestVersion)
+	if err1 == nil && err2 == nil {
+		s.hasUpdate = currVersion.LessThan(latestVersion)
+	}
+}
+
+func (s *Service) GrafanaUpdateAvailable() bool {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return s.hasUpdate
+}
+
+func (s *Service) LatestGrafanaVersion() string {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return s.latestVersion
+}