Plugins: Refactor Plugin Management (#40477)

* add core plugin flow

* add instrumentation

* move func

* remove cruft

* support external backend plugins

* refactor + clean up

* remove comments

* refactor loader

* simplify core plugin path arg

* cleanup loggers

* move signature validator to plugins package

* fix sig packaging

* cleanup plugin model

* remove unnecessary plugin field

* add start+stop for pm

* fix failures

* add decommissioned state

* export fields just to get things flowing

* fix comments

* set static routes

* make image loading idempotent

* merge with backend plugin manager

* re-use funcs

* reorder imports + remove unnecessary interface

* add some TODOs + remove unused func

* remove unused instrumentation func

* simplify client usage

* remove import alias

* re-use backendplugin.Plugin interface

* re order funcs

* improve var name

* fix log statements

* refactor data model

* add logic for dupe check during loading

* cleanup state setting

* refactor loader

* cleanup manager interface

* add rendering flow

* refactor loading + init

* add renderer support

* fix renderer plugin

* reformat imports

* track errors

* fix plugin signature inheritance

* name param in interface

* update func comment

* fix func arg name

* introduce class concept

* remove func

* fix external plugin check

* apply changes from pm-experiment

* fix core plugins

* fix imports

* rename interface

* comment API interface

* add support for testdata plugin

* enable alerting + use correct core plugin contracts

* slim manager API

* fix param name

* fix filter

* support static routes

* fix rendering

* tidy rendering

* get tests compiling

* fix install+uninstall

* start finder test

* add finder test coverage

* start loader tests

* add test for core plugins

* load core + bundled test

* add test for nested plugin loading

* add test files

* clean interface + fix registering some core plugins

* refactoring

* reformat and create sub packages

* simplify core plugin init

* fix ctx cancel scenario

* migrate initializer

* remove Init() funcs

* add test starter

* new logger

* flesh out initializer tests

* refactoring

* remove unused svc

* refactor rendering flow

* fixup loader tests

* add enabled helper func

* fix logger name

* fix data fetchers

* fix case where plugin dir doesn't exist

* improve coverage + move dupe checking to loader

* remove noisy debug logs

* register core plugins automagically

* add support for renderer in catalog

* make private func + fix req validation

* use interface

* re-add check for renderer in catalog

* tidy up from moving to auto reg core plugins

* core plugin registrar

* guards

* copy over core plugins for test infra

* all tests green

* renames

* propagate new interfaces

* kill old manager

* get compiling

* tidy up

* update naming

* refactor manager test + cleanup

* add more cases to finder test

* migrate validator to field

* more coverage

* refactor dupe checking

* add test for plugin class

* add coverage for initializer

* split out rendering

* move

* fixup tests

* fix uss test

* fix frontend settings

* fix grafanads test

* add check when checking sig errors

* fix enabled map

* fixup

* allow manual setup of CM

* rename to cloud-monitoring

* remove TODO

* add installer interface for testing

* loader interface returns

* tests passing

* refactor + add more coverage

* support 'stackdriver'

* fix frontend settings loading

* improve naming based on package name

* small tidy

* refactor test

* fix renderer start

* make cloud-monitoring plugin ID clearer

* add plugin update test

* add integration tests

* don't break all if sig can't be calculated

* add root URL check test

* add more signature verification tests

* update DTO name

* update enabled plugins comment

* update comments

* fix linter

* revert fe naming change

* fix errors endpoint

* reset error code field name

* re-order test to help verify

* assert -> require

* pm check

* add missing entry + re-order

* re-check

* dump icon log

* verify manager contents first

* reformat

* apply PR feedback

* apply style changes

* fix one vs all loading err

* improve log output

* only start when no signature error

* move log

* rework plugin update check

* fix test

* fix multi loading from cfg.PluginSettings

* improve log output #2

* add error abstraction to capture errors without registering a plugin

* add debug log

* add unsigned warning

* e2e test attempt

* fix logger

* set home path

* prevent panic

* alternate

* ugh.. fix home path

* return renderer even if not started

* make renderer plugin managed

* add fallback renderer icon, update renderer badge + prevent changes when renderer is installed

* fix icon loading

* rollback renderer changes

* use correct field

* remove unneccessary block

* remove newline

* remove unused func

* fix bundled plugins base + module fields

* remove unused field since refactor

* add authorizer abstraction

* loader only returns plugins expected to run

* fix multi log output

pkg/tsdb/testdatasource/stream_handler.go

@@ -9,34 +9,11 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-
-	"github.com/grafana/grafana/pkg/infra/log"
 )
 
-type testStreamHandler struct {
-	logger log.Logger
-	frame  *data.Frame
-	// If Live Pipeline enabled we are sending the whole frame to have a chance to process stream with rules.
-	livePipelineEnabled bool
-}
-
-func newTestStreamHandler(logger log.Logger, livePipelineEnabled bool) *testStreamHandler {
-	frame := data.NewFrame("testdata",
-		data.NewField("Time", nil, make([]time.Time, 1)),
-		data.NewField("Value", nil, make([]float64, 1)),
-		data.NewField("Min", nil, make([]float64, 1)),
-		data.NewField("Max", nil, make([]float64, 1)),
-	)
-	return &testStreamHandler{
-		frame:               frame,
-		logger:              logger,
-		livePipelineEnabled: livePipelineEnabled,
-	}
-}
-
-func (p *testStreamHandler) SubscribeStream(_ context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) {
-	p.logger.Debug("Allowing access to stream", "path", req.Path, "user", req.PluginContext.User)
-	initialData, err := backend.NewInitialFrame(p.frame, data.IncludeSchemaOnly)
+func (s *Service) SubscribeStream(_ context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) {
+	s.logger.Debug("Allowing access to stream", "path", req.Path, "user", req.PluginContext.User)
+	initialData, err := backend.NewInitialFrame(s.frame, data.IncludeSchemaOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -50,7 +27,7 @@ func (p *testStreamHandler) SubscribeStream(_ context.Context, req *backend.Subs
 		}
 	}
 
-	if p.livePipelineEnabled {
+	if s.cfg.FeatureToggles["live-pipeline"] {
 		// While developing Live pipeline avoid sending initial data.
 		initialData = nil
 	}
@@ -61,15 +38,15 @@ func (p *testStreamHandler) SubscribeStream(_ context.Context, req *backend.Subs
 	}, nil
 }
 
-func (p *testStreamHandler) PublishStream(_ context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) {
-	p.logger.Debug("Attempt to publish into stream", "path", req.Path, "user", req.PluginContext.User)
+func (s *Service) PublishStream(_ context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) {
+	s.logger.Debug("Attempt to publish into stream", "path", req.Path, "user", req.PluginContext.User)
 	return &backend.PublishStreamResponse{
 		Status: backend.PublishStreamStatusPermissionDenied,
 	}, nil
 }
 
-func (p *testStreamHandler) RunStream(ctx context.Context, request *backend.RunStreamRequest, sender *backend.StreamSender) error {
-	p.logger.Debug("New stream call", "path", request.Path)
+func (s *Service) RunStream(ctx context.Context, request *backend.RunStreamRequest, sender *backend.StreamSender) error {
+	s.logger.Debug("New stream call", "path", request.Path)
 	var conf testStreamConfig
 	switch request.Path {
 	case "random-2s-stream":
@@ -93,7 +70,7 @@ func (p *testStreamHandler) RunStream(ctx context.Context, request *backend.RunS
 	default:
 		return fmt.Errorf("testdata plugin does not support path: %s", request.Path)
 	}
-	return p.runTestStream(ctx, request.Path, conf, sender)
+	return s.runTestStream(ctx, request.Path, conf, sender)
 }
 
 type testStreamConfig struct {
@@ -102,7 +79,7 @@ type testStreamConfig struct {
 	Flight   *flightConfig
 }
 
-func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf testStreamConfig, sender *backend.StreamSender) error {
+func (s *Service) runTestStream(ctx context.Context, path string, conf testStreamConfig, sender *backend.StreamSender) error {
 	spread := 50.0
 	walker := rand.Float64() * 100
 
@@ -118,7 +95,7 @@ func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf
 	for {
 		select {
 		case <-ctx.Done():
-			p.logger.Debug("Stop streaming data for path", "path", path)
+			s.logger.Debug("Stop streaming data for path", "path", path)
 			return ctx.Err()
 		case t := <-ticker.C:
 			if rand.Float64() < conf.Drop {
@@ -126,7 +103,7 @@ func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf
 			}
 
 			mode := data.IncludeDataOnly
-			if p.livePipelineEnabled {
+			if s.cfg.FeatureToggles["live-pipeline"] {
 				mode = data.IncludeAll
 			}
 
@@ -139,11 +116,11 @@ func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf
 				delta := rand.Float64() - 0.5
 				walker += delta
 
-				p.frame.Fields[0].Set(0, t)
-				p.frame.Fields[1].Set(0, walker)                                // Value
-				p.frame.Fields[2].Set(0, walker-((rand.Float64()*spread)+0.01)) // Min
-				p.frame.Fields[3].Set(0, walker+((rand.Float64()*spread)+0.01)) // Max
-				if err := sender.SendFrame(p.frame, mode); err != nil {
+				s.frame.Fields[0].Set(0, t)
+				s.frame.Fields[1].Set(0, walker)                                // Value
+				s.frame.Fields[2].Set(0, walker-((rand.Float64()*spread)+0.01)) // Min
+				s.frame.Fields[3].Set(0, walker+((rand.Float64()*spread)+0.01)) // Max
+				if err := sender.SendFrame(s.frame, mode); err != nil {
 					return err
 				}
 			}