Plugins: Add preinstall_sync config - allow plugins to be preinstalled synchronously (#105292)

* feat: preinstall_sync config - process and installation logic

* ref: add preinstall_sync list to preinstalled plugins of frontendsettings

* fix: conf blank line for sections

* ref: remove plugins async flag, and rename PreinstallPlugins

* docs: default installed plugin list
This commit is contained in:
Syerikjan Kh
2025-05-16 10:45:50 -04:00
committed by GitHub
parent 7e038b67a5
commit b38662e985
12 changed files with 200 additions and 84 deletions
+4 -3
View File
@@ -1837,10 +1837,11 @@ hide_angular_deprecation =
# Comma separated list of plugin ids for which environment variables should be forwarded. Used only when feature flag pluginsSkipHostEnvVars is enabled.
forward_host_env_vars =
# Comma separated list of plugin ids to install as part of the startup process.
# By default, the following plugins will be preinstalled: "grafana-lokiexplore-app"
# These will be installed, by default, asynchronously (in the background) while starting Grafana.
preinstall =
# Controls whether preinstall plugins asynchronously (in the background) or synchronously (blocking). Useful when preinstalled plugins are used with provisioning.
preinstall_async = true
# Comma separated list of plugin ids to install before the startup process
# These will be installed before starting Grafana. Useful when used with provisioning.
preinstall_sync =
# Disables preinstall feature. It has the same effect as setting preinstall to an empty list.
preinstall_disabled = false
# Update strategy for plugins.
+8
View File
@@ -1782,6 +1782,14 @@ default_datasource_uid =
; public_key_retrieval_on_startup = false
# Enter a comma-separated list of plugin identifiers to avoid loading (including core plugins). These plugins will be hidden in the catalog.
; disable_plugins =
# Comma separated list of plugin ids to install as part of the startup process.
# These will be installed, by default, asynchronously (in the background) while starting Grafana.
; preinstall =
# Comma separated list of plugin ids to install before the startup process
# These will be installed before starting Grafana. Useful when used with provisioning.
; preinstall_sync =
# Disables preinstall feature. It has the same effect as setting preinstall to an empty list.
; preinstall_disabled = false
#################################### Grafana Live ##########################################
[live]
@@ -2492,18 +2492,21 @@ These plugins are hidden in the catalog.
Enter a comma-separated list of plugin identifiers to install on startup, using the Grafana catalog as the source.
Preinstalled plugins cannot be uninstalled from the Grafana user interface; they need to be removed from this list first.
Plugins are installed asynchronously, as a background process.
This means that Grafana starts up faster, but the plugins may not be available immediately.
To pin plugins to a specific version, use the format `plugin_id@version`, for example,`grafana-piechart-panel@1.6.0`. If no version is specified, the latest version is installed. _The plugin is automatically updated_ to the latest version when a new version is available in the Grafana plugin catalog on startup (except for new major versions).
To use a custom URL to download a plugin, use the format `plugin_id@version@url`, for example, `grafana-piechart-panel@1.6.0@https://example.com/grafana-piechart-panel-1.6.0.zip`.
By default, Grafana installs some suggested plugins on startup. Refer to the default configuration file for that list of plugins.
By default, Grafana installs some suggested plugins on startup. For a list of default preinstalled plugins, refer to [pkg/setting/setting_plugins.go:35](https://github.com/grafana/grafana/blob/main/pkg/setting/setting_plugins.go#L35-L40).
#### `preinstall_async`
#### `preinstall_sync`
By default, plugins are preinstalled asynchronously, as a background process.
This means that Grafana starts up faster, but the plugins may not be available immediately.
If you need a plugin to be installed for provisioning, set this option to `false`.
This causes Grafana to wait for the plugins to be installed before starting up and fail if a plugin can't be installed.
Enter a comma-separated list of plugin identifiers to install on startup, using the Grafana catalog as the source.
Same as `preinstall`, but installs plugins synchronously.
These will be installed before starting Grafana. Useful when used with provisioning.
#### `preinstall_disabled`
+1 -1
View File
@@ -298,7 +298,7 @@ func (hs *HTTPServer) getFrontendSettings(c *contextmodel.ReqContext) (*dtos.Fro
PluginAdminExternalManageEnabled: hs.Cfg.PluginAdminEnabled && hs.Cfg.PluginAdminExternalManageEnabled,
PluginCatalogHiddenPlugins: hs.Cfg.PluginCatalogHiddenPlugins,
PluginCatalogManagedPlugins: hs.managedPluginsService.ManagedPlugins(c.Req.Context()),
PluginCatalogPreinstalledPlugins: hs.Cfg.PreinstallPlugins,
PluginCatalogPreinstalledPlugins: append(hs.Cfg.PreinstallPluginsAsync, hs.Cfg.PreinstallPluginsSync...),
ExpressionsEnabled: hs.Cfg.ExpressionsEnabled,
AwsAllowedAuthProviders: hs.Cfg.AWSAllowedAuthProviders,
AwsAssumeRoleEnabled: hs.Cfg.AWSAssumeRoleEnabled,
+1 -1
View File
@@ -101,7 +101,7 @@ func Test_PluginsInstallAndUninstall(t *testing.T) {
hs.Cfg.PluginAdminEnabled = tc.pluginAdminEnabled
hs.Cfg.PluginAdminExternalManageEnabled = tc.pluginAdminExternalManageEnabled
hs.Cfg.RBAC.SingleOrganization = tc.singleOrganization
hs.Cfg.PreinstallPlugins = []setting.InstallPlugin{{ID: "grafana-preinstalled-datasource", Version: "1.0.0"}}
hs.Cfg.PreinstallPluginsAsync = []setting.InstallPlugin{{ID: "grafana-preinstalled-datasource", Version: "1.0.0"}}
hs.orgService = &orgtest.FakeOrgService{ExpectedOrg: &org.Org{}}
hs.accesscontrolService = &actest.FakeService{}
@@ -11,7 +11,7 @@ func ProvidePreinstall(
cfg *setting.Cfg,
) *PreinstallImpl {
plugins := make(map[string]*setting.InstallPlugin)
for _, p := range cfg.PreinstallPlugins {
for _, p := range cfg.PreinstallPluginsAsync {
plugins[p.ID] = &p
}
return &PreinstallImpl{
@@ -9,7 +9,7 @@ import (
func TestIsPreinstalled(t *testing.T) {
cfg := &setting.Cfg{
PreinstallPlugins: []setting.InstallPlugin{
PreinstallPluginsAsync: []setting.InstallPlugin{
{ID: "plugin1"},
{ID: "plugin2"},
},
@@ -23,7 +23,7 @@ func TestIsPreinstalled(t *testing.T) {
func TestIsPinned(t *testing.T) {
cfg := &setting.Cfg{
PreinstallPlugins: []setting.InstallPlugin{
PreinstallPluginsAsync: []setting.InstallPlugin{
{ID: "plugin1", Version: "1.0.0"},
{ID: "plugin2"},
},
@@ -42,7 +42,6 @@ type Service struct {
pluginStore pluginstore.Store
pluginRepo repo.Service
features featuremgmt.FeatureToggles
failOnErr bool
updateChecker pluginchecker.PluginUpdateChecker
}
@@ -65,14 +64,13 @@ func ProvideService(
cfg: cfg,
pluginInstaller: pluginInstaller,
pluginStore: pluginStore,
failOnErr: !cfg.PreinstallPluginsAsync, // Fail on error if preinstall is synchronous
pluginRepo: pluginRepo,
features: features,
updateChecker: updateChecker,
}
if !cfg.PreinstallPluginsAsync {
if len(cfg.PreinstallPluginsSync) > 0 {
// Block initialization process until plugins are installed
err := s.installPluginsWithTimeout()
err := s.installPluginsWithTimeout(cfg.PreinstallPluginsSync)
if err != nil {
return nil, err
}
@@ -82,11 +80,10 @@ func ProvideService(
// IsDisabled disables background installation of plugins.
func (s *Service) IsDisabled() bool {
return len(s.cfg.PreinstallPlugins) == 0 ||
!s.cfg.PreinstallPluginsAsync
return len(s.cfg.PreinstallPluginsAsync) == 0
}
func (s *Service) installPluginsWithTimeout() error {
func (s *Service) installPluginsWithTimeout(pluginsToInstall []setting.InstallPlugin) error {
// Installation process does not timeout by default nor reuses the context
// passed to the request so we need to handle the timeout here.
// We could make this timeout configurable in the future.
@@ -94,7 +91,7 @@ func (s *Service) installPluginsWithTimeout() error {
defer cancel()
done := make(chan struct{ err error })
go func() {
done <- struct{ err error }{err: s.installPlugins(ctx)}
done <- struct{ err error }{err: s.installPlugins(ctx, pluginsToInstall, true)}
}()
select {
case <-ctx.Done():
@@ -114,8 +111,8 @@ func (s *Service) shouldUpdate(ctx context.Context, pluginID, currentVersion str
return s.updateChecker.CanUpdate(pluginID, currentVersion, info.Version, true)
}
func (s *Service) installPlugins(ctx context.Context) error {
for _, installPlugin := range s.cfg.PreinstallPlugins {
func (s *Service) installPlugins(ctx context.Context, pluginsToInstall []setting.InstallPlugin, failOnErr bool) error {
for _, installPlugin := range pluginsToInstall {
// Check if the plugin is already installed
p, exists := s.pluginStore.Plugin(ctx, installPlugin.ID)
if exists {
@@ -148,7 +145,7 @@ func (s *Service) installPlugins(ctx context.Context) error {
s.log.Debug("Plugin already installed", "pluginId", installPlugin.ID, "version", installPlugin.Version)
continue
}
if s.failOnErr {
if failOnErr {
// Halt execution in the synchronous scenario
return fmt.Errorf("failed to install plugin %s@%s: %w", installPlugin.ID, installPlugin.Version, err)
}
@@ -165,7 +162,7 @@ func (s *Service) installPlugins(ctx context.Context) error {
}
func (s *Service) Run(ctx context.Context) error {
err := s.installPlugins(ctx)
err := s.installPlugins(ctx, s.cfg.PreinstallPluginsAsync, false)
if err != nil {
// Unexpected error, asynchronous installation should not return errors
s.log.Error("Failed to install plugins", "error", err)
@@ -24,8 +24,7 @@ func TestService_IsDisabled(t *testing.T) {
// Create a new service
s, err := ProvideService(
&setting.Cfg{
PreinstallPlugins: []setting.InstallPlugin{{ID: "myplugin"}},
PreinstallPluginsAsync: true,
PreinstallPluginsAsync: []setting.InstallPlugin{{ID: "myplugin"}},
},
pluginstore.New(registry.NewInMemory(), &fakes.FakeLoader{}),
&fakes.FakePluginInstaller{},
@@ -44,13 +43,14 @@ func TestService_IsDisabled(t *testing.T) {
func TestService_Run(t *testing.T) {
tests := []struct {
name string
shouldInstall bool
pluginsToInstall []setting.InstallPlugin
existingPlugins []*plugins.Plugin
pluginsToFail []string
blocking bool
latestPlugin *repo.PluginArchiveInfo
name string
shouldInstall bool
shouldThrowError bool
pluginsToInstall []setting.InstallPlugin
pluginsToInstallSync []setting.InstallPlugin
existingPlugins []*plugins.Plugin
pluginsToFail []string
latestPlugin *repo.PluginArchiveInfo
}{
{
name: "Installs a plugin",
@@ -64,6 +64,7 @@ func TestService_Run(t *testing.T) {
},
{
name: "Skips already installed plugin",
shouldThrowError: false,
shouldInstall: false,
pluginsToInstall: []setting.InstallPlugin{{ID: "myplugin"}},
existingPlugins: []*plugins.Plugin{{JSONData: plugins.JSONData{ID: "myplugin"}}},
@@ -86,17 +87,16 @@ func TestService_Run(t *testing.T) {
pluginsToFail: []string{"myplugin1"},
},
{
name: "Install a blocking plugin",
shouldInstall: true,
pluginsToInstall: []setting.InstallPlugin{{ID: "myplugin"}},
blocking: true,
name: "Install a plugin from sync list",
shouldInstall: true,
pluginsToInstallSync: []setting.InstallPlugin{{ID: "myplugin"}},
},
{
name: "Fails to install a blocking plugin",
shouldInstall: false,
pluginsToInstall: []setting.InstallPlugin{{ID: "myplugin"}},
blocking: true,
pluginsToFail: []string{"myplugin"},
name: "when installation fails in sync mode, it should throw an error",
shouldInstall: false,
shouldThrowError: true,
pluginsToInstallSync: []setting.InstallPlugin{{ID: "myplugin"}},
pluginsToFail: []string{"myplugin"},
},
{
name: "Updates a plugin",
@@ -138,6 +138,12 @@ func TestService_Run(t *testing.T) {
existingPlugins: []*plugins.Plugin{{JSONData: plugins.JSONData{ID: "myplugin", Info: plugins.Info{Version: "1.0.0"}}}},
latestPlugin: &repo.PluginArchiveInfo{Version: "1.0.0-rc.1"},
},
{
name: "should install all plugins - sync and async",
shouldInstall: true,
pluginsToInstallSync: []setting.InstallPlugin{{ID: "myplugin"}},
pluginsToInstall: []setting.InstallPlugin{{ID: "myplugin2"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -150,8 +156,8 @@ func TestService_Run(t *testing.T) {
installedFromURL := 0
s, err := ProvideService(
&setting.Cfg{
PreinstallPlugins: tt.pluginsToInstall,
PreinstallPluginsAsync: !tt.blocking,
PreinstallPluginsAsync: tt.pluginsToInstall,
PreinstallPluginsSync: tt.pluginsToInstallSync,
},
pluginstore.New(preg, &fakes.FakeLoader{}),
&fakes.FakePluginInstaller{
@@ -165,7 +171,8 @@ func TestService_Run(t *testing.T) {
t.Fatal("Should not install plugin")
return errors.New("Should not install plugin")
}
for _, plugin := range tt.pluginsToInstall {
allPluginsToInstall := append(tt.pluginsToInstallSync, tt.pluginsToInstall...)
for _, plugin := range allPluginsToInstall {
if plugin.ID == pluginID && plugin.Version == version {
if opts.URL() != "" {
installedFromURL++
@@ -190,20 +197,19 @@ func TestService_Run(t *testing.T) {
&pluginchecker.FakePluginPreinstall{},
),
)
if tt.blocking && !tt.shouldInstall {
if tt.shouldThrowError {
require.ErrorContains(t, err, "Failed to install plugin")
} else {
require.NoError(t, err)
return
}
require.NoError(t, err)
err = s.Run(context.Background())
require.NoError(t, err)
if !tt.blocking {
err = s.Run(context.Background())
require.NoError(t, err)
}
if tt.shouldInstall {
expectedInstalled := 0
expectedInstalledFromURL := 0
for _, plugin := range tt.pluginsToInstall {
allPluginsToInstall := append(tt.pluginsToInstallSync, tt.pluginsToInstall...)
for _, plugin := range allPluginsToInstall {
expectedFailed := false
for _, pluginFail := range tt.pluginsToFail {
if plugin.ID == pluginFail {
+2 -2
View File
@@ -200,8 +200,8 @@ type Cfg struct {
DisablePlugins []string
HideAngularDeprecation []string
ForwardHostEnvVars []string
PreinstallPlugins []InstallPlugin
PreinstallPluginsAsync bool
PreinstallPluginsAsync []InstallPlugin
PreinstallPluginsSync []InstallPlugin
PluginsCDNURLTemplate string
PluginLogBackendRequests bool
+44 -23
View File
@@ -41,6 +41,24 @@ var (
}
)
func (cfg *Cfg) processPreinstallPlugins(rawInstallPlugins []string, preinstallPlugins map[string]InstallPlugin) {
// Add the plugins defined in the configuration
for _, plugin := range rawInstallPlugins {
parts := strings.Split(plugin, "@")
id := parts[0]
version := ""
url := ""
if len(parts) > 1 {
version = parts[1]
if len(parts) > 2 {
url = parts[2]
}
}
preinstallPlugins[id] = InstallPlugin{id, version, url}
}
}
func (cfg *Cfg) readPluginSettings(iniFile *ini.File) error {
pluginsSection := iniFile.Section("plugins")
@@ -56,38 +74,41 @@ func (cfg *Cfg) readPluginSettings(iniFile *ini.File) error {
cfg.ForwardHostEnvVars = util.SplitString(pluginsSection.Key("forward_host_env_vars").MustString(""))
disablePreinstall := pluginsSection.Key("preinstall_disabled").MustBool(false)
if !disablePreinstall {
rawInstallPlugins := util.SplitString(pluginsSection.Key("preinstall").MustString(""))
preinstallPlugins := make(map[string]InstallPlugin)
// Add the default preinstalled plugins
rawInstallPluginsAsync := util.SplitString(pluginsSection.Key("preinstall").MustString(""))
preinstallPluginsAsync := make(map[string]InstallPlugin)
// Add the default preinstalled plugins to pre install plugins async list
for _, plugin := range defaultPreinstallPlugins {
preinstallPlugins[plugin.ID] = plugin
preinstallPluginsAsync[plugin.ID] = plugin
}
if cfg.IsFeatureToggleEnabled("grafanaAdvisor") { // Use literal string to avoid circular dependency
preinstallPlugins["grafana-advisor-app"] = InstallPlugin{"grafana-advisor-app", "", ""}
preinstallPluginsAsync["grafana-advisor-app"] = InstallPlugin{"grafana-advisor-app", "", ""}
}
// Add the plugins defined in the configuration
for _, plugin := range rawInstallPlugins {
parts := strings.Split(plugin, "@")
id := parts[0]
version := ""
url := ""
if len(parts) > 1 {
version = parts[1]
if len(parts) > 2 {
url = parts[2]
}
}
cfg.processPreinstallPlugins(rawInstallPluginsAsync, preinstallPluginsAsync)
preinstallPlugins[id] = InstallPlugin{id, version, url}
}
rawInstallPluginsSync := util.SplitString(pluginsSection.Key("preinstall_sync").MustString(""))
preinstallPluginsSync := make(map[string]InstallPlugin)
cfg.processPreinstallPlugins(rawInstallPluginsSync, preinstallPluginsSync)
// Remove from the list the plugins that have been disabled
for _, disabledPlugin := range cfg.DisablePlugins {
delete(preinstallPlugins, disabledPlugin)
delete(preinstallPluginsAsync, disabledPlugin)
delete(preinstallPluginsSync, disabledPlugin)
}
for _, plugin := range preinstallPlugins {
cfg.PreinstallPlugins = append(cfg.PreinstallPlugins, plugin)
for _, plugin := range preinstallPluginsAsync {
cfg.PreinstallPluginsAsync = append(cfg.PreinstallPluginsAsync, plugin)
}
for _, plugin := range preinstallPluginsSync {
cfg.PreinstallPluginsSync = append(cfg.PreinstallPluginsSync, plugin)
}
installPluginsInAsync := pluginsSection.Key("preinstall_async").MustBool(true)
if !installPluginsInAsync {
for key, plugin := range preinstallPluginsAsync {
if _, exists := preinstallPluginsSync[key]; !exists {
cfg.PreinstallPluginsSync = append(cfg.PreinstallPluginsSync, plugin)
}
}
cfg.PreinstallPluginsAsync = nil
}
cfg.PreinstallPluginsAsync = pluginsSection.Key("preinstall_async").MustBool(true)
}
cfg.PluginCatalogURL = pluginsSection.Key("plugin_catalog_url").MustString("https://grafana.com/grafana/plugins/")
+84 -4
View File
@@ -97,6 +97,56 @@ func Test_readPluginSettings(t *testing.T) {
}
})
t.Run("when plugins.preinstall_sync is defined", func(t *testing.T) {
tests := []struct {
name string
rawInput string
expected []InstallPlugin
disablePlugins string
disablePreinstall bool
}{
{
name: "should add the plugin to the sync list - not contain default plugins like async list",
rawInput: "plugin1",
expected: []InstallPlugin{
{ID: "plugin1", Version: "", URL: ""},
},
},
{
name: "it should remove the disabled plugin",
rawInput: "plugin1,plugin2",
disablePlugins: "plugin1",
expected: []InstallPlugin{{ID: "plugin2"}},
},
{
name: "should not process at all when preinstall is disabled",
rawInput: "plugin1",
disablePreinstall: true,
expected: nil,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := NewCfg()
sec, err := cfg.Raw.NewSection("plugins")
require.NoError(t, err)
_, err = sec.NewKey("preinstall_sync", tc.rawInput)
require.NoError(t, err)
if tc.disablePreinstall {
_, err = sec.NewKey("preinstall_disabled", "true")
require.NoError(t, err)
}
if tc.disablePlugins != "" {
_, err = sec.NewKey("disable_plugins", tc.disablePlugins)
require.NoError(t, err)
}
err = cfg.readPluginSettings(cfg.Raw)
require.NoError(t, err)
assert.ElementsMatch(t, cfg.PreinstallPluginsSync, tc.expected)
})
}
})
t.Run("when plugins.preinstall is defined", func(t *testing.T) {
defaultPreinstallPluginsList := make([]InstallPlugin, 0, len(defaultPreinstallPlugins))
defaultPreinstallPluginsIDs := []string{}
@@ -107,8 +157,10 @@ func Test_readPluginSettings(t *testing.T) {
tests := []struct {
name string
rawInput string
rawInputSync string
disablePreinstall bool
expected []InstallPlugin
expectedSync []InstallPlugin
disableAsync bool
disablePlugins string
}{
@@ -149,7 +201,8 @@ func Test_readPluginSettings(t *testing.T) {
name: "should mark preinstall as sync",
rawInput: "plugin1",
disableAsync: true,
expected: append(defaultPreinstallPluginsList, InstallPlugin{"plugin1", "", ""}),
expected: nil,
expectedSync: append(defaultPreinstallPluginsList, InstallPlugin{"plugin1", "", ""}),
},
{
name: "should parse a plugin with version and URL",
@@ -161,6 +214,29 @@ func Test_readPluginSettings(t *testing.T) {
rawInput: "plugin1@@https://example.com/plugin1.tar.gz",
expected: append(defaultPreinstallPluginsList, InstallPlugin{"plugin1", "", "https://example.com/plugin1.tar.gz"}),
},
{
name: "when preinstall_async is false, should add all plugins to preinstall_sync",
rawInput: "plugin1",
rawInputSync: "plugin2",
disableAsync: true,
expected: nil,
expectedSync: append(defaultPreinstallPluginsList, InstallPlugin{"plugin1", "", ""}, InstallPlugin{"plugin2", "", ""}),
},
{
name: "should overwrite default when user pins a version",
rawInput: "grafana-pyroscope-app@4.0.0",
expected: func() []InstallPlugin {
var plugins []InstallPlugin
for _, p := range defaultPreinstallPlugins {
if p.ID == "grafana-pyroscope-app" {
plugins = append(plugins, InstallPlugin{"grafana-pyroscope-app", "4.0.0", ""})
} else {
plugins = append(plugins, p)
}
}
return plugins
}(),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
@@ -169,6 +245,10 @@ func Test_readPluginSettings(t *testing.T) {
require.NoError(t, err)
_, err = sec.NewKey("preinstall", tc.rawInput)
require.NoError(t, err)
if tc.rawInputSync != "" {
_, err = sec.NewKey("preinstall_sync", tc.rawInputSync)
require.NoError(t, err)
}
if tc.disablePreinstall {
_, err = sec.NewKey("preinstall_disabled", "true")
require.NoError(t, err)
@@ -184,9 +264,9 @@ func Test_readPluginSettings(t *testing.T) {
err = cfg.readPluginSettings(cfg.Raw)
require.NoError(t, err)
assert.ElementsMatch(t, cfg.PreinstallPlugins, tc.expected)
if tc.disableAsync {
require.Equal(t, cfg.PreinstallPluginsAsync, false)
assert.ElementsMatch(t, cfg.PreinstallPluginsAsync, tc.expected)
if tc.expectedSync != nil {
assert.ElementsMatch(t, cfg.PreinstallPluginsSync, tc.expectedSync)
}
})
}