admin api: Fix reencryption of private_key in signing_key table, introduce test for reencryption (#105010)

* Add integration test for reencrypting secrets.
Rotate keys before reencryption.
Test for setting table from enterprise repo.

pkg/api/admin_encryption.go

@@ -25,7 +25,7 @@ func (hs *HTTPServer) AdminReEncryptEncryptionKeys(c *contextmodel.ReqContext) r
 }
 
 func (hs *HTTPServer) AdminReEncryptSecrets(c *contextmodel.ReqContext) response.Response {
-	success, err := hs.secretsMigrator.ReEncryptSecrets(c.Req.Context())
+	success, err := hs.SecretsMigrator.ReEncryptSecrets(c.Req.Context())
 	if err != nil {
 		return response.Error(http.StatusInternalServerError, "Failed to re-encrypt secrets", err)
 	}
@@ -38,7 +38,7 @@ func (hs *HTTPServer) AdminReEncryptSecrets(c *contextmodel.ReqContext) response
 }
 
 func (hs *HTTPServer) AdminRollbackSecrets(c *contextmodel.ReqContext) response.Response {
-	success, err := hs.secretsMigrator.RollBackSecrets(c.Req.Context())
+	success, err := hs.SecretsMigrator.RollBackSecrets(c.Req.Context())
 	if err != nil {
 		return response.Error(http.StatusInternalServerError, "Failed to rollback secrets", err)
 	}

pkg/api/http_server.go

@@ -170,7 +170,7 @@ type HTTPServer struct {
 	EncryptionService            encryption.Internal
 	SecretsService               secrets.Service
 	secretsStore                 secretsKV.SecretsKVStore
-	secretsMigrator              secrets.Migrator
+	SecretsMigrator              secrets.Migrator
 	secretMigrationProvider      spm.SecretMigrationProvider
 	DataSourcesService           datasources.DataSourceService
 	cleanUpService               *cleanup.CleanUpService
@@ -328,7 +328,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		SocialService:                socialService,
 		EncryptionService:            encryptionService,
 		SecretsService:               secretsService,
-		secretsMigrator:              secretsMigrator,
+		SecretsMigrator:              secretsMigrator,
 		secretMigrationProvider:      secretMigrationProvider,
 		secretsStore:                 secretsStore,
 		DataSourcesService:           dataSourcesService,

pkg/services/secrets/migrator/migrator.go

@@ -14,7 +14,9 @@ import (
 )
 
 type SecretsRotator interface {
+	// ReEncrypt returns true on success, false on any failure.
 	ReEncrypt(context.Context, *manager.SecretsService, db.DB) bool
+	// Rollback returns true on success, false on any failure.
 	Rollback(context.Context, *manager.SecretsService, encryption.Internal, db.DB, string) bool
 }
 
@@ -43,7 +45,7 @@ func ProvideSecretsMigrator(
 		b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
 		jsonSecret{tableName: "data_source"},
 		jsonSecret{tableName: "plugin_setting"},
-		b64Secret{simpleSecret: simpleSecret{tableName: "signing_key", columnName: "private_key"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "signing_key", columnName: "private_key"}, encoding: base64.RawStdEncoding},
 		alertingSecret{},
 		ssoSettingsSecret{},
 		b64Secret{simpleSecret: simpleSecret{tableName: "user_external_session", columnName: "access_token"}, encoding: base64.StdEncoding},
@@ -94,12 +96,12 @@ func (m *SecretsMigrator) RollBackSecrets(ctx context.Context) (bool, error) {
 	var anyFailure bool
 
 	for _, r := range m.rotators {
-		if failed := r.Rollback(ctx,
+		if success := r.Rollback(ctx,
 			m.secretsSrv,
 			m.encryptionSrv,
 			m.sqlStore,
 			m.settings.KeyValue("security", "secret_key").Value(),
-		); failed {
+		); !success {
 			anyFailure = true
 		}
 	}

pkg/services/secrets/migrator/provisioning.go

@@ -101,7 +101,7 @@ func (p provisioningSecrets) reEncrypt(
 	} else {
 		logger.Info("Successfully rotated provisioning secrets", "action", action)
 	}
-	return failures > 0
+	return failures == 0
 }
 
 func (provisioningSecrets) reEncryptGitHubToken(

pkg/services/secrets/migrator/reencrypt.go

@@ -25,7 +25,7 @@ func (s simpleSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.Secrets
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows)
 	}); err != nil {
-		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName, "error", err)
 		return false
 	}
 
@@ -84,7 +84,7 @@ func (s b64Secret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsSer
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows)
 	}); err != nil {
-		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName, "error", err)
 		return false
 	}
 
@@ -155,7 +155,7 @@ func (s jsonSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsSe
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Table(s.tableName).Cols("id", "secure_json_data").Find(&rows)
 	}); err != nil {
-		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName, "error", err)
 		return false
 	}
 
@@ -219,7 +219,7 @@ func (s alertingSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.Secre
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.SQL(selectSQL).Find(&results)
 	}); err != nil {
-		logger.Warn("Could not find any alert_configuration secret to re-encrypt")
+		logger.Warn("Could not find any alert_configuration secret to re-encrypt", "error", err)
 		return false
 	}
 
@@ -299,9 +299,8 @@ func (s ssoSettingsSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.Se
 	err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Find(&results)
 	})
-
 	if err != nil {
-		logger.Warn("Failed to fetch SSO settings to re-encrypt", "err", err)
+		logger.Warn("Failed to fetch SSO settings to re-encrypt", "error", err)
 		return false
 	}
 

pkg/services/secrets/migrator/rollback.go

@@ -19,7 +19,7 @@ func (s simpleSecret) Rollback(
 	encryptionSrv encryption.Internal,
 	sqlStore db.DB,
 	secretKey string,
-) (anyFailure bool) {
+) bool {
 	var rows []struct {
 		Id     int
 		Secret []byte
@@ -28,10 +28,11 @@ func (s simpleSecret) Rollback(
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows)
 	}); err != nil {
-		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
-		return true
+		logger.Warn("Could not find any secret to roll back", "table", s.tableName, "error", err)
+		return false
 	}
 
+	anyFailure := false
 	for _, row := range rows {
 		if len(row.Secret) == 0 {
 			continue
@@ -70,7 +71,7 @@ func (s simpleSecret) Rollback(
 		logger.Info(fmt.Sprintf("Column %s from %s has been rolled back successfully", s.columnName, s.tableName))
 	}
 
-	return anyFailure
+	return !anyFailure
 }
 
 func (s b64Secret) Rollback(
@@ -79,7 +80,7 @@ func (s b64Secret) Rollback(
 	encryptionSrv encryption.Internal,
 	sqlStore db.DB,
 	secretKey string,
-) (anyFailure bool) {
+) bool {
 	var rows []struct {
 		Id     int
 		Secret string
@@ -89,9 +90,10 @@ func (s b64Secret) Rollback(
 		return sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows)
 	}); err != nil {
 		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
-		return true
+		return false
 	}
 
+	anyFailure := false
 	for _, row := range rows {
 		if len(row.Secret) == 0 {
 			continue
@@ -144,7 +146,7 @@ func (s b64Secret) Rollback(
 		logger.Info(fmt.Sprintf("Column %s from %s has been rolled back successfully", s.columnName, s.tableName))
 	}
 
-	return anyFailure
+	return !anyFailure
 }
 
 func (s jsonSecret) Rollback(
@@ -153,7 +155,7 @@ func (s jsonSecret) Rollback(
 	encryptionSrv encryption.Internal,
 	sqlStore db.DB,
 	secretKey string,
-) (anyFailure bool) {
+) bool {
 	var rows []struct {
 		Id             int
 		SecureJsonData map[string][]byte
@@ -162,10 +164,11 @@ func (s jsonSecret) Rollback(
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Table(s.tableName).Cols("id", "secure_json_data").Find(&rows)
 	}); err != nil {
-		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
-		return true
+		logger.Warn("Could not find any secret to roll back", "table", s.tableName, "error", err)
+		return false
 	}
 
+	anyFailure := false
 	for _, row := range rows {
 		if len(row.SecureJsonData) == 0 {
 			continue
@@ -208,7 +211,7 @@ func (s jsonSecret) Rollback(
 		logger.Info(fmt.Sprintf("Secure json data secrets from %s have been rolled back successfully", s.tableName))
 	}
 
-	return anyFailure
+	return !anyFailure
 }
 
 func (s alertingSecret) Rollback(
@@ -217,7 +220,7 @@ func (s alertingSecret) Rollback(
 	encryptionSrv encryption.Internal,
 	sqlStore db.DB,
 	secretKey string,
-) (anyFailure bool) {
+) bool {
 	var results []struct {
 		Id                        int
 		AlertmanagerConfiguration string
@@ -227,17 +230,16 @@ func (s alertingSecret) Rollback(
 	if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.SQL(selectSQL).Find(&results)
 	}); err != nil {
-		logger.Warn("Could not find any alert_configuration secret to roll back")
-		return true
+		logger.Warn("Could not find any alert_configuration secret to roll back", "error", err)
+		return false
 	}
 
+	anyFailure := false
 	for _, result := range results {
-		result := result
-
 		err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
 			postableUserConfig, err := notifier.Load([]byte(result.AlertmanagerConfiguration))
 			if err != nil {
-				logger.Warn("Could not load configuration (alert_configuration with id: %d) while rolling it back", result.Id, err)
+				logger.Warn("Could not load configuration while rolling it back", "id", result.Id, "error", err)
 				return err
 			}
 
@@ -246,19 +248,19 @@ func (s alertingSecret) Rollback(
 					for k, v := range gmr.SecureSettings {
 						decoded, err := base64.StdEncoding.DecodeString(v)
 						if err != nil {
-							logger.Warn("Could not decode base64-encoded secret (alert_configuration with id: %d, key)", k, result.Id, err)
+							logger.Warn("Could not decode base64-encoded secret", "id", result.Id, "key", k, "error", err)
 							return err
 						}
 
 						decrypted, err := secretsSrv.Decrypt(ctx, decoded)
 						if err != nil {
-							logger.Warn("Could not decrypt secret (alert_configuration with id: %d, key)", k, result.Id, err)
+							logger.Warn("Could not decrypt secret", "id", result.Id, "key", k, "error", err)
 							return err
 						}
 
 						reencrypted, err := encryptionSrv.Encrypt(ctx, decrypted, secretKey)
 						if err != nil {
-							logger.Warn("Could not re-encrypt secret (alert_configuration with id: %d, key)", k, result.Id, err)
+							logger.Warn("Could not re-encrypt secret", "id", result.Id, "key", k, "error", err)
 							return err
 						}
 
@@ -269,13 +271,13 @@ func (s alertingSecret) Rollback(
 
 			marshalled, err := json.Marshal(postableUserConfig)
 			if err != nil {
-				logger.Warn("Could not marshal configuration (alert_configuration with id: %d) while rolling it back", result.Id, err)
+				logger.Warn("Could not marshal configuration while rolling it back", "id", result.Id, "error", err)
 				return err
 			}
 
 			result.AlertmanagerConfiguration = string(marshalled)
 			if _, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result); err != nil {
-				logger.Warn("Could not update secret (alert_configuration with id: %d) while rolling it back", result.Id, err)
+				logger.Warn("Could not update secret while rolling it back", "id", result.Id, "error", err)
 				return err
 			}
 
@@ -293,7 +295,7 @@ func (s alertingSecret) Rollback(
 		logger.Info("Alerting configuration secrets have been rolled back successfully")
 	}
 
-	return anyFailure
+	return !anyFailure
 }
 
 func (s ssoSettingsSecret) Rollback(
@@ -302,18 +304,18 @@ func (s ssoSettingsSecret) Rollback(
 	encryptionSrv encryption.Internal,
 	sqlStore db.DB,
 	secretKey string,
-) (anyFailure bool) {
+) bool {
 	results := make([]*models.SSOSettings, 0)
 
 	err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
 		return sess.Find(&results)
 	})
-
 	if err != nil {
-		logger.Warn("Failed to fetch SSO settings to roll back")
-		return true
+		logger.Warn("Failed to fetch SSO settings to roll back", "error", err)
+		return false
 	}
 
+	anyFailure := false
 	for _, result := range results {
 		err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
 			result.Settings, err = s.reEncryptSecretsInMap(ctx, result.Settings, secretsSrv, encryptionSrv, secretKey)
@@ -345,5 +347,5 @@ func (s ssoSettingsSecret) Rollback(
 		logger.Info("SSO settings secrets have been rolled back successfully")
 	}
 
-	return anyFailure
+	return !anyFailure
 }

pkg/storage/unified/sql/db/dbimpl/dbimpl.go

@@ -7,7 +7,6 @@ import (
 	"sync"
 
 	"github.com/dlmiddlecote/sqlstats"
-	"github.com/grafana/grafana/pkg/util/xorm"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/trace"
 	"go.opentelemetry.io/otel/trace/noop"
@@ -18,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db/migrations"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db/otel"
+	"github.com/grafana/grafana/pkg/util/xorm"
 )
 
 const (
@@ -104,10 +104,6 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
 		return p, fmt.Errorf("invalid db type specified: %s", dbType)
 
 	// If we have an empty Resource API db config, try with the core Grafana database config
-	case grafanaDBType != "":
-		p.registerMetrics = true
-		p.engine, err = getEngine(cfg)
-		return p, err
 	case grafanaDB != nil:
 		// try to use the grafana db connection (should only happen in tests)
 		if fallbackGetter.Bool(grafanaDBInstrumentQueriesKey) {
@@ -115,6 +111,10 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
 		}
 		p.engine = grafanaDB.GetEngine()
 		return p, nil
+	case grafanaDBType != "":
+		p.registerMetrics = true
+		p.engine, err = getEngine(cfg)
+		return p, err
 	default:
 		return p, fmt.Errorf("no database type specified")
 	}

pkg/tests/api/admin/encryption/reencrypt_enterprise_test.go

@@ -0,0 +1,97 @@
+//go:build enterprise
+// +build enterprise
+
+package encryption
+
+import (
+	"encoding/base64"
+	"fmt"
+	"hash/fnv"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/server"
+	"github.com/grafana/grafana/pkg/services/org"
+	"github.com/grafana/grafana/pkg/services/user"
+)
+
+func TestIntegration_AdminApiReencrypt_Enterprise(t *testing.T) {
+	getSecretsFunctions := map[string]func(*testing.T, *server.TestEnv) map[int]secret{}
+	getSecretsFunctions["settings"] = func(t *testing.T, env *server.TestEnv) map[int]secret {
+		return getSettingSecrets(t, env.SQLStore)
+	}
+
+	setup := func(t *testing.T, env *server.TestEnv, grafanaListenAddr string) {
+		createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
+			DefaultOrgRole: string(org.RoleAdmin),
+			Password:       "admin",
+			Login:          "admin",
+			IsAdmin:        true,
+		})
+
+		addSetting(t, grafanaListenAddr)
+	}
+
+	RunAdminApiReencryptTest(t, setup, getSecretsFunctions)
+}
+
+func addSetting(t *testing.T, grafanaListenAddr string) {
+	body := `
+		{
+		  "updates": {
+			"auth.saml": {
+			  "enabled": "true",
+			  "single_logout": "false"
+			}
+		  }
+		}
+	`
+
+	url := fmt.Sprintf("http://admin:admin@%s/api/admin/settings", grafanaListenAddr)
+
+	req, err := http.NewRequest("PUT", url, strings.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func getSettingSecrets(t *testing.T, store db.DB) map[int]secret {
+	var rows []struct {
+		Section        string
+		Key            string
+		EncryptedValue string
+	}
+
+	err := store.WithDbSession(t.Context(), func(sess *db.Session) error {
+		return sess.Table("setting").Select("section, key, encrypted_value").OrderBy("section, key").Find(&rows)
+	})
+	require.NoError(t, err)
+
+	result := map[int]secret{}
+	for _, r := range rows {
+		d, err := base64.StdEncoding.DecodeString(r.EncryptedValue)
+		require.NoError(t, err)
+
+		// we don't care about hash collisions, as long as we use the same ordering, we check the same values.
+		id := int(hash(r.Section + ":" + r.Key))
+		result[id] = secret{
+			id:     id,
+			secret: d,
+			// no update time
+		}
+	}
+	return result
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(s))
+	return h.Sum32()
+}

pkg/tests/api/admin/encryption/reencrypt_test.go

@@ -0,0 +1,339 @@
+package encryption
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	claims "github.com/grafana/authlib/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/server"
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/services/authn"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/grafana/grafana/pkg/services/org"
+	"github.com/grafana/grafana/pkg/services/org/orgimpl"
+	"github.com/grafana/grafana/pkg/services/quota/quotaimpl"
+	"github.com/grafana/grafana/pkg/services/supportbundles/supportbundlestest"
+	"github.com/grafana/grafana/pkg/services/user"
+	"github.com/grafana/grafana/pkg/services/user/userimpl"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/tests/testinfra"
+	"github.com/grafana/grafana/pkg/tests/testsuite"
+)
+
+func TestMain(m *testing.M) {
+	testsuite.Run(m)
+}
+
+func TestIntegration_AdminApiReencrypt(t *testing.T) {
+	const (
+		dataSourceTable              = "data_source"
+		secretsTable                 = "secrets"
+		secretsValueColumn           = "value"
+		alertmanagerSecureSettingKey = "secure-value"
+		secureJsonKey                = "db-secure-key"
+	)
+
+	getSecretsFunctions := map[string]func(*testing.T, *server.TestEnv) map[int]secret{}
+	getSecretsFunctions["secureJson-"+dataSourceTable] = func(t *testing.T, env *server.TestEnv) map[int]secret {
+		return getSecureJsonSecrets(t, env.SQLStore, dataSourceTable, secureJsonKey)
+	}
+	getSecretsFunctions["base64-"+secretsTable+"-"+secretsValueColumn] = func(t *testing.T, env *server.TestEnv) map[int]secret {
+		return getBase64Secrets(t, env.SQLStore, secretsTable, secretsValueColumn, base64.RawStdEncoding)
+	}
+	getSecretsFunctions["alertmanager"] = func(t *testing.T, env *server.TestEnv) map[int]secret {
+		return getAlertmanagerSecrets(t, env.SQLStore, alertmanagerSecureSettingKey)
+	}
+	getSecretsFunctions["signing_keys"] = func(t *testing.T, env *server.TestEnv) map[int]secret {
+		return getSigningKeys(t, env.SQLStore)
+	}
+
+	setup := func(t *testing.T, env *server.TestEnv, grafanaListenAddr string) {
+		userId := createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
+			DefaultOrgRole: string(org.RoleAdmin),
+			Password:       "admin",
+			Login:          "admin",
+		})
+
+		dsCmd := &datasources.AddDataSourceCommand{
+			Name:            "TestDatasource",
+			Type:            "testdata",
+			Access:          datasources.DS_ACCESS_DIRECT,
+			UID:             "testuid",
+			UserID:          userId,
+			OrgID:           1,
+			WithCredentials: true,
+			SecureJsonData: map[string]string{
+				secureJsonKey: "db-secure-value",
+			},
+		}
+		// This creates secret both in `data_source` table and `secrets` table.
+		_, err := env.Server.HTTPServer.DataSourcesService.AddDataSource(context.Background(), dsCmd)
+		require.NoError(t, err)
+
+		// Trigger creation of signing key
+		_, _, err = env.IDService.SignIdentity(context.Background(), &authn.Identity{ID: fmt.Sprintf("%d", userId), Type: claims.TypeUser})
+		require.NoError(t, err)
+
+		// Add alerting config with secure settings.
+		addAlertingConfig(t, env)
+	}
+
+	RunAdminApiReencryptTest(t, setup, getSecretsFunctions)
+}
+
+// This test verifies that secrets in various databases are reencrypted with new data key when reencryption is triggered.
+// Setup function is supposed to create various secrets that are then
+// obtained via "secretsFunctions".
+// This test is quite generic so that it can be called from enterprise repository as well.
+func RunAdminApiReencryptTest(
+	t *testing.T,
+	setup func(t *testing.T, env *server.TestEnv, grafanaListenAddr string),
+	secretsFns map[string]func(t *testing.T, env *server.TestEnv) map[int]secret,
+) {
+	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		APIServerStorageType: options.StorageTypeUnified,
+	})
+
+	grafanaListenAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
+	setup(t, env, grafanaListenAddr)
+
+	beforeReencrypt := getSecrets(t, secretsFns, env)
+
+	err := env.Server.HTTPServer.SecretsService.RotateDataKeys(context.Background())
+	require.NoError(t, err)
+
+	// Reencrypt with new data key.
+	ok, err := env.Server.HTTPServer.SecretsMigrator.ReEncryptSecrets(context.Background())
+	require.NoError(t, err)
+	assert.True(t, ok, "Failed to reencrypt all secrets")
+
+	afterReencrypt := getSecrets(t, secretsFns, env)
+	verifyAllSecrets(t, env, beforeReencrypt, afterReencrypt)
+
+	// Rollback from envelope to legacy encryption.
+	ok, err = env.Server.HTTPServer.SecretsMigrator.RollBackSecrets(context.Background())
+	require.NoError(t, err)
+	assert.True(t, ok, "Failed to rollback all secrets")
+
+	afterRollback := getSecrets(t, secretsFns, env)
+	verifyAllSecrets(t, env, afterReencrypt, afterRollback)
+}
+
+func getSecrets(t *testing.T, secretsFunctions map[string]func(t *testing.T, env *server.TestEnv) map[int]secret, env *server.TestEnv) map[string]map[int]secret {
+	secrets := map[string]map[int]secret{}
+	for name, fn := range secretsFunctions {
+		s := fn(t, env)
+		require.NotEmpty(t, s, "Failed to get secrets from function %s", name)
+		secrets[name] = s
+	}
+	return secrets
+}
+
+func getAlertmanagerSecrets(t *testing.T, store db.DB, secureSettingKey string) map[int]secret {
+	var rows []struct {
+		Id                        int
+		AlertmanagerConfiguration string
+	}
+	err := store.WithDbSession(t.Context(), func(sess *db.Session) error {
+		return sess.Table("alert_configuration").Cols("id", "alertmanager_configuration").Find(&rows)
+	})
+	require.NoError(t, err)
+
+	result := map[int]secret{}
+
+next:
+	for _, r := range rows {
+		postableUserConfig, err := notifier.Load([]byte(r.AlertmanagerConfiguration))
+		require.NoError(t, err)
+
+		// Find first grafana-managed receiver config with secure settings with given key, and extract it.
+		for _, receiver := range postableUserConfig.AlertmanagerConfig.Receivers {
+			for _, gmr := range receiver.GrafanaManagedReceivers {
+				v := gmr.SecureSettings[secureSettingKey]
+				if v == "" {
+					continue
+				}
+
+				decoded, err := base64.StdEncoding.DecodeString(v)
+				require.NoError(t, err)
+				result[r.Id] = secret{
+					id:     r.Id,
+					secret: decoded,
+				}
+				continue next
+			}
+		}
+	}
+	return result
+}
+
+func addAlertingConfig(t *testing.T, env *server.TestEnv) {
+	// Create alertmanager config
+	cfg := apimodels.PostableUserConfig{}
+	body := `
+		{
+			"alertmanager_config": {
+				"route": {
+					"receiver": "grafana-default-email"
+				},
+				"receivers": [{
+					"name": "grafana-default-email",
+					"grafana_managed_receiver_configs": [{
+						"uid": "",
+						"name": "email receiver",
+						"type": "email",
+						"isDefault": true,
+						"settings": {
+							"addresses": "<example@email.com>"
+						},
+						"secureSettings": {
+							"secure-value": "secret"
+						}
+					}]
+				}]
+			}
+		}
+		`
+	err := json.Unmarshal([]byte(body), &cfg)
+	require.NoError(t, err)
+	err = env.Server.HTTPServer.AlertNG.MultiOrgAlertmanager.SaveAndApplyAlertmanagerConfiguration(context.Background(), 1, cfg)
+	require.NoError(t, err)
+}
+
+type secret struct {
+	id     int
+	secret []byte
+	update time.Time
+}
+
+func verifyAllSecrets(t *testing.T, env *server.TestEnv, before, after map[string]map[int]secret) {
+	require.Equal(t, len(before), len(after))
+	for k, bef := range before {
+		aft, ok := after[k]
+		require.True(t, ok)
+		verifySecrets(t, env, bef, aft)
+	}
+}
+
+func verifySecrets(t *testing.T, env *server.TestEnv, before, after map[int]secret) {
+	require.Equal(t, len(before), len(after))
+	for k, bef := range before {
+		aft, ok := after[k]
+		require.True(t, ok, "key not found: %d", k)
+
+		require.NotEmpty(t, bef.secret, "before secret is empty for key %d", k)
+		require.NotEmpty(t, aft.secret, "after secret is empty for key %d", k)
+		require.NotEqual(t, bef.secret, aft.secret, "secrets are equal after reencrypt for key %d", k)
+
+		s1, err := env.Server.HTTPServer.SecretsService.Decrypt(context.Background(), bef.secret)
+		require.NoError(t, err)
+		s2, err := env.Server.HTTPServer.SecretsService.Decrypt(context.Background(), aft.secret)
+		require.NoError(t, err)
+		assert.Equal(t, string(s1), string(s2), "decrypted secrets are not equal for key %d", k)
+
+		updatedDiff := aft.update.Sub(bef.update)
+		// Since we're storing timestamps with seconds resolution, diff can be 0.
+		require.True(t, 0 <= updatedDiff && updatedDiff <= time.Minute, "Updated time difference (%v) outside of allowed range for key %d", updatedDiff, k)
+	}
+}
+
+func createUser(t *testing.T, db db.DB, cfg *setting.Cfg, cmd user.CreateUserCommand) int64 {
+	cfg.AutoAssignOrg = true
+	cfg.AutoAssignOrgId = 1
+
+	quotaService := quotaimpl.ProvideService(db, cfg)
+	orgService, err := orgimpl.ProvideService(db, cfg, quotaService)
+	require.NoError(t, err)
+	usrSvc, err := userimpl.ProvideService(
+		db, orgService, cfg, nil, nil, tracing.InitializeTracerForTest(),
+		quotaService, supportbundlestest.NewFakeBundleService(),
+	)
+	require.NoError(t, err)
+
+	u, err := usrSvc.Create(context.Background(), &cmd)
+	require.NoError(t, err)
+	return u.ID
+}
+
+func getSecureJsonSecrets(t *testing.T, store db.DB, table string, secureJsonDataKey string) map[int]secret {
+	var rows []struct {
+		Id             int
+		SecureJsonData map[string][]byte
+		Updated        time.Time
+	}
+
+	err := store.WithDbSession(t.Context(), func(sess *db.Session) error {
+		return sess.Table(table).Cols("id", "secure_json_data", "updated").Find(&rows)
+	})
+	require.NoError(t, err)
+
+	result := map[int]secret{}
+	for _, r := range rows {
+		result[r.Id] = secret{
+			id:     r.Id,
+			secret: r.SecureJsonData[secureJsonDataKey],
+			update: r.Updated,
+		}
+	}
+	return result
+}
+
+func getBase64Secrets(t *testing.T, store db.DB, table, column string, enc *base64.Encoding) map[int]secret {
+	var rows []struct {
+		Id      int
+		Secret  string
+		Updated time.Time
+	}
+
+	err := store.WithDbSession(t.Context(), func(sess *db.Session) error {
+		return sess.Table(table).Select(fmt.Sprintf("id, %s as secret, updated", column)).Find(&rows)
+	})
+	require.NoError(t, err)
+
+	result := map[int]secret{}
+	for _, r := range rows {
+		d, err := enc.DecodeString(r.Secret)
+		require.NoError(t, err)
+		result[r.Id] = secret{
+			id:     r.Id,
+			secret: d,
+			update: r.Updated,
+		}
+	}
+	return result
+}
+
+func getSigningKeys(t *testing.T, store db.DB) map[int]secret {
+	var rows []struct {
+		Id int
+		Pk string
+	}
+
+	err := store.WithDbSession(t.Context(), func(sess *db.Session) error {
+		return sess.Table("signing_key").Select("id, private_key as pk").Find(&rows)
+	})
+	require.NoError(t, err)
+
+	result := map[int]secret{}
+	for _, r := range rows {
+		d, err := base64.RawStdEncoding.DecodeString(r.Pk)
+		require.NoError(t, err)
+		result[r.Id] = secret{
+			id:     r.Id,
+			secret: d,
+			// there's no update time, leave it at 0
+		}
+	}
+	return result
+}