Plugins: Add synchronous CDN plugin loader (#99096)

* WIP

* Run plugin validations and validation steps sequentially if feature is off

* Remove dependency between sources.Service and pluginscdn.Service

* lint

* Parallelize validation only if class is CDN

* re-generate feature toggles

* remove waitgroup usage

* PR review: Add loader concurrency limit setting

* re-generate feature toggles

* pr review feedback

* fix const name

* Skip module.js validation for cdn plugins

* do not run validation steps in parallel

* lint

* reduce diff

* re-generate feature toggles

* lint

* pr review feedback

* remove leftover config.PluginManagementCfg from sources.Service

pkg/services/pluginsintegration/loader/loader.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
 	pluginsLoader "github.com/grafana/grafana/pkg/plugins/manager/loader"
 	"github.com/grafana/grafana/pkg/plugins/manager/pipeline/bootstrap"
 	"github.com/grafana/grafana/pkg/plugins/manager/pipeline/discovery"
@@ -19,11 +20,13 @@ type Loader struct {
 	loader *pluginsLoader.Loader
 }
 
-func ProvideService(discovery discovery.Discoverer, bootstrap bootstrap.Bootstrapper, validation validation.Validator,
+func ProvideService(
+	cfg *config.PluginManagementCfg,
+	discovery discovery.Discoverer, bootstrap bootstrap.Bootstrapper, validation validation.Validator,
 	initializer initialization.Initializer, termination termination.Terminator, errorTracker pluginerrs.ErrorTracker,
 ) *Loader {
 	return &Loader{
-		loader: pluginsLoader.New(discovery, bootstrap, validation, initializer, termination, errorTracker),
+		loader: pluginsLoader.New(cfg, discovery, bootstrap, validation, initializer, termination, errorTracker),
 	}
 }
 

pkg/services/pluginsintegration/loader/loader_test.go

@@ -1575,7 +1575,7 @@ func newLoader(t *testing.T, cfg *config.PluginManagementCfg, reg registry.Servi
 	terminate, err := pipeline.ProvideTerminationStage(cfg, reg, proc)
 	require.NoError(t, err)
 
-	return ProvideService(pipeline.ProvideDiscoveryStage(cfg,
+	return ProvideService(cfg, pipeline.ProvideDiscoveryStage(cfg,
 		finder.NewLocalFinder(false), reg),
 		pipeline.ProvideBootstrapStage(cfg, signature.DefaultCalculator(cfg), assets),
 		pipeline.ProvideValidationStage(cfg, signature.NewValidator(signature.NewUnsignedAuthorizer(cfg)), angularInspector),
@@ -1607,7 +1607,7 @@ func newLoaderWithOpts(t *testing.T, cfg *config.PluginManagementCfg, opts loade
 		backendFactoryProvider = fakes.NewFakeBackendProcessProvider()
 	}
 
-	return ProvideService(pipeline.ProvideDiscoveryStage(cfg,
+	return ProvideService(cfg, pipeline.ProvideDiscoveryStage(cfg,
 		finder.NewLocalFinder(false), reg),
 		pipeline.ProvideBootstrapStage(cfg, signature.DefaultCalculator(cfg), assets),
 		pipeline.ProvideValidationStage(cfg, signature.NewValidator(signature.NewUnsignedAuthorizer(cfg)), angularInspector),

pkg/services/pluginsintegration/pluginconfig/config.go

@@ -30,9 +30,10 @@ func ProvidePluginManagementConfig(cfg *setting.Cfg, settingProvider setting.Pro
 		cfg.PluginsCDNURLTemplate,
 		cfg.AppURL,
 		config.Features{
-			ExternalCorePluginsEnabled: features.IsEnabledGlobally(featuremgmt.FlagExternalCorePlugins),
-			SkipHostEnvVarsEnabled:     features.IsEnabledGlobally(featuremgmt.FlagPluginsSkipHostEnvVars),
-			SriChecksEnabled:           features.IsEnabledGlobally(featuremgmt.FlagPluginsSriChecks),
+			ExternalCorePluginsEnabled:  features.IsEnabledGlobally(featuremgmt.FlagExternalCorePlugins),
+			SkipHostEnvVarsEnabled:      features.IsEnabledGlobally(featuremgmt.FlagPluginsSkipHostEnvVars),
+			SriChecksEnabled:            features.IsEnabledGlobally(featuremgmt.FlagPluginsSriChecks),
+			PluginsCDNSyncLoaderEnabled: features.IsEnabledGlobally(featuremgmt.FlagPluginsCDNSyncLoader),
 		},
 		cfg.AngularSupportEnabled,
 		cfg.GrafanaComAPIURL,

pkg/services/pluginsintegration/pluginsintegration.go

@@ -107,8 +107,6 @@ var WireSet = wire.NewSet(
 	wire.Bind(new(repo.Service), new(*repo.Manager)),
 	licensing.ProvideLicensing,
 	wire.Bind(new(plugins.Licensing), new(*licensing.Service)),
-	wire.Bind(new(sources.Registry), new(*sources.Service)),
-	sources.ProvideService,
 	pluginSettings.ProvideService,
 	wire.Bind(new(pluginsettings.Service), new(*pluginSettings.Service)),
 	filestore.ProvideService,
@@ -146,6 +144,8 @@ var WireExtensionSet = wire.NewSet(
 	wire.Bind(new(plugins.Client), new(*backend.MiddlewareHandler)),
 	managedplugins.NewNoop,
 	wire.Bind(new(managedplugins.Manager), new(*managedplugins.Noop)),
+	sources.ProvideService,
+	wire.Bind(new(sources.Registry), new(*sources.Service)),
 )
 
 func ProvideClientWithMiddlewares(

pkg/services/pluginsintegration/renderer/renderer.go

@@ -141,5 +141,5 @@ func createLoader(cfg *config.PluginManagementCfg, pluginEnvProvider envvars.Pro
 
 	et := pluginerrs.ProvideErrorTracker()
 
-	return loader.New(d, b, v, i, t, et), nil
+	return loader.New(cfg, d, b, v, i, t, et), nil
 }

pkg/services/pluginsintegration/test_helper.go

@@ -110,5 +110,5 @@ func CreateTestLoader(t *testing.T, cfg *pluginsCfg.PluginManagementCfg, opts Lo
 		require.NoError(t, err)
 	}
 
-	return loader.New(opts.Discoverer, opts.Bootstrapper, opts.Validator, opts.Initializer, opts.Terminator, pluginerrs.ProvideErrorTracker())
+	return loader.New(cfg, opts.Discoverer, opts.Bootstrapper, opts.Validator, opts.Initializer, opts.Terminator, pluginerrs.ProvideErrorTracker())
 }