Secrets: Implement migration of secrets from plugin back to unified secrets (#53561)

* initial cut at migration from plugin

* create new migration from plugin

* only migrate to or from, not both

* remove cfg check from plugin migration itself

* update comments, clean up secret after migration

* add better error handling

* hook up REST API with migrations

* Minor fixes

* fix wire injection issue

* modify migrator to access plugin calls directly. create unit tests

* change pre-migration checks in admin api

* stop plugin after migrating from it

* fix compile issues after merge

* add comment about migration

* fix linting issue

* bleh, fix unit test

* fix another unit test

* update plugin error fatal flag after a migration from the plugin

* add extra logging to migration

* make linter happy

Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>

pkg/api/admin_encryption.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/models"
+	skv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 )
 
 func (hs *HTTPServer) AdminRotateDataEncryptionKeys(c *models.ReqContext) response.Response {
@@ -48,3 +49,33 @@ func (hs *HTTPServer) AdminRollbackSecrets(c *models.ReqContext) response.Respon
 
 	return response.Respond(http.StatusOK, "Secrets rolled back successfully")
 }
+
+// To migrate to the plugin, it must be installed and configured
+// so as not to lose access to migrated secrets
+func (hs *HTTPServer) MigrateSecretsToPlugin(c *models.ReqContext) response.Response {
+	if skv.EvaluateRemoteSecretsPlugin(hs.secretsPluginManager, hs.Cfg) != nil {
+		hs.log.Warn("Received secrets plugin migration request while plugin is not available")
+		return response.Respond(http.StatusBadRequest, "Secrets plugin is not available")
+	}
+	err := hs.secretsPluginMigrator.TriggerPluginMigration(c.Req.Context(), true)
+	if err != nil {
+		hs.log.Error("Failed to trigger secret migration to plugin", "error", err.Error())
+		return response.Respond(http.StatusInternalServerError, "Secret migration to plugin failed")
+	}
+	return response.Respond(http.StatusOK, "Secret migration to plugin triggered successfully")
+}
+
+// To migrate from the plugin, it must be installed only
+// as it is possible the user disabled it and then wants to migrate
+func (hs *HTTPServer) MigrateSecretsFromPlugin(c *models.ReqContext) response.Response {
+	if hs.secretsPluginManager.SecretsManager() == nil {
+		hs.log.Warn("Received secrets plugin migration request while plugin is not installed")
+		return response.Respond(http.StatusBadRequest, "Secrets plugin is not installed")
+	}
+	err := hs.secretsPluginMigrator.TriggerPluginMigration(c.Req.Context(), false)
+	if err != nil {
+		hs.log.Error("Failed to trigger secret migration from plugin", "error", err.Error())
+		return response.Respond(http.StatusInternalServerError, "Secret migration from plugin failed")
+	}
+	return response.Respond(http.StatusOK, "Secret migration from plugin triggered successfully")
+}

pkg/api/api.go

@@ -599,6 +599,8 @@ func (hs *HTTPServer) registerRoutes() {
 		adminRoute.Post("/encryption/reencrypt-data-keys", reqGrafanaAdmin, routing.Wrap(hs.AdminReEncryptEncryptionKeys))
 		adminRoute.Post("/encryption/reencrypt-secrets", reqGrafanaAdmin, routing.Wrap(hs.AdminReEncryptSecrets))
 		adminRoute.Post("/encryption/rollback-secrets", reqGrafanaAdmin, routing.Wrap(hs.AdminRollbackSecrets))
+		adminRoute.Post("/encryption/migrate-secrets/to-plugin", reqGrafanaAdmin, routing.Wrap(hs.MigrateSecretsToPlugin))
+		adminRoute.Post("/encryption/migrate-secrets/from-plugin", reqGrafanaAdmin, routing.Wrap(hs.MigrateSecretsFromPlugin))
 
 		adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards))
 		adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins))

pkg/api/http_server.go

@@ -76,6 +76,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/search"
 	"github.com/grafana/grafana/pkg/services/searchusers"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	spm "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations"
 	"github.com/grafana/grafana/pkg/services/serviceaccounts"
 	"github.com/grafana/grafana/pkg/services/shorturls"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
@@ -178,6 +179,7 @@ type HTTPServer struct {
 	apiKeyService                apikey.Service
 	kvStore                      kvstore.KVStore
 	secretsMigrator              secrets.Migrator
+	secretsPluginMigrator        *spm.SecretMigrationServiceImpl
 	userService                  user.Service
 	tempUserService              tempUser.Service
 	loginAttemptService          loginAttempt.Service
@@ -217,7 +219,8 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	teamsPermissionsService accesscontrol.TeamPermissionsService, folderPermissionsService accesscontrol.FolderPermissionsService,
 	dashboardPermissionsService accesscontrol.DashboardPermissionsService, dashboardVersionService dashver.Service,
 	starService star.Service, csrfService csrf.Service, coremodels *registry.Base,
-	playlistService playlist.Service, apiKeyService apikey.Service, kvStore kvstore.KVStore, secretsMigrator secrets.Migrator, secretsPluginManager plugins.SecretsPluginManager,
+	playlistService playlist.Service, apiKeyService apikey.Service, kvStore kvstore.KVStore,
+	secretsMigrator secrets.Migrator, secretsPluginManager plugins.SecretsPluginManager, secretsPluginMigrator *spm.SecretMigrationServiceImpl,
 	publicDashboardsApi *publicdashboardsApi.Api, userService user.Service, tempUserService tempUser.Service, loginAttemptService loginAttempt.Service, orgService org.Service,
 	accesscontrolService accesscontrol.Service,
 ) (*HTTPServer, error) {
@@ -307,6 +310,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		kvStore:                      kvStore,
 		PublicDashboardsApi:          publicDashboardsApi,
 		secretsMigrator:              secretsMigrator,
+		secretsPluginMigrator:        secretsPluginMigrator,
 		userService:                  userService,
 		tempUserService:              tempUserService,
 		loginAttemptService:          loginAttemptService,