[release-12.1.6] Fix primary keys (#115638)

* [release-12.1.6] Fix primary keys Co-authored-by: grafana-delivery-bot[bot] <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Co-authored-by: Will Assis <35489495+gassiss@users.noreply.github.com> * Run migrations even if feature flag is turned off. We need to migrate all tables in order to generate DB schema dump. * Skip tests that are broken due to incompatible DB schema. --------- Co-authored-by: grafana-delivery-bot[bot] <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Co-authored-by: Will Assis <35489495+gassiss@users.noreply.github.com>
2026-01-06 10:06:55 +01:00
parent cbd5e4d4a9
commit 10a08f907e
15 changed files with 632 additions and 32 deletions
@@ -19,6 +19,14 @@ func RegisterDependencies(
 	secretDBMigrator contracts.SecretDBMigrator,
 	accessControlService accesscontrol.Service,
 ) (*DependencyRegisterer, error) {
+	// Some DBs that claim to be MySQL/Postgres-compatible might not support table locking.
+	lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true)
+
+	// This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator.
+	if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil {
+		return nil, fmt.Errorf("running secret database migrations: %w", err)
+	}
+
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) || !features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
 		return nil, nil
 	}
@@ -28,16 +36,5 @@ func RegisterDependencies(
 		return nil, fmt.Errorf("registering access control roles: %w", err)
 	}

-	// We shouldn't need to create the DB in HG, as that will use the MT api server.
-	if cfg.StackID == "" {
-		// Some DBs that claim to be MySQL/Postgres-compatible might not support table locking.
-		lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true)
-
-		// This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator.
-		if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil {
-			return nil, fmt.Errorf("running secret database migrations: %w", err)
-		}
-	}
-
 	return &DependencyRegisterer{}, nil
 }
@@ -26,6 +26,8 @@ func TestMain(m *testing.M) {
 }

 func Test_SQLKeeperSetup(t *testing.T) {
+	t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2")
+
 	ctx := context.Background()
 	namespace1 := "namespace1"
 	namespace2 := "namespace2"
@@ -4,11 +4,14 @@ import (
 	"context"
 	"testing"

-	secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
-	encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
 	"go.opentelemetry.io/otel/trace/noop"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

+	secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
+	encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
+
+	"github.com/stretchr/testify/require"
+
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/decrypt"
@@ -25,7 +28,6 @@ import (
 	"github.com/grafana/grafana/pkg/storage/secret/database"
 	"github.com/grafana/grafana/pkg/storage/secret/metadata"
 	"github.com/grafana/grafana/pkg/storage/secret/migrator"
-	"github.com/stretchr/testify/require"
 )

 type SetupConfig struct {
@@ -50,6 +52,8 @@ func WithMutateCfg(f func(*SetupConfig)) func(*SetupConfig) {
 }

 func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
+	t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2")
+
 	setupCfg := defaultSetupCfg()
 	for _, opt := range opts {
 		opt(&setupCfg)
@@ -187,4 +187,33 @@ func addCloudMigrationsMigrations(mg *Migrator) {
 	mg.AddMigration("increase resource_uid column length", NewRawSQLMigration("").
 		Mysql("ALTER TABLE cloud_migration_resource MODIFY resource_uid NVARCHAR(255);").
 		Postgres("ALTER TABLE cloud_migration_resource ALTER COLUMN resource_uid TYPE VARCHAR(255);"))
+
+	migrationSnapshotPartitionTable := Table{
+		Name: "cloud_migration_snapshot_partition",
+		Columns: []*Column{
+			{Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false},
+			{Name: "partition_number", Type: DB_Int, Nullable: false},
+			{Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false},
+			{Name: "data", Type: DB_LongBlob, Nullable: false},
+		},
+	}
+
+	mg.AddMigration("create cloud_migration_snapshot_partition table v1", NewAddTableMigration(migrationSnapshotPartitionTable))
+	srpUniqueIndex := Index{
+		Name: "srp_unique",
+		Cols: []string{"snapshot_uid", "resource_type", "partition_number"}, Type: UniqueIndex,
+	}
+	mg.AddMigration("add cloud_migration_snapshot_partition srp_unique index", NewAddIndexMigration(migrationSnapshotPartitionTable, &srpUniqueIndex))
+
+	updatedCloudMigrationSnapshotPartitionTable := Table{
+		Name: "cloud_migration_snapshot_partition",
+		Columns: []*Column{
+			{Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true},
+			{Name: "partition_number", Type: DB_Int, Nullable: false, IsPrimaryKey: true},
+			{Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false, IsPrimaryKey: true},
+			{Name: "data", Type: DB_LongBlob, Nullable: false},
+		},
+		PrimaryKeys: []string{"snapshot_uid", "resource_type", "partition_number"},
+	}
+	ConvertUniqueKeyToPrimaryKey(mg, srpUniqueIndex, updatedCloudMigrationSnapshotPartitionTable)
 }
@@ -67,4 +67,139 @@ func addDbFileStorageMigration(mg *migrator.Migrator) {

 	mg.AddMigration("migrate contents column to mediumblob for MySQL", migrator.NewRawSQLMigration("").
 		Mysql("ALTER TABLE file MODIFY contents MEDIUMBLOB;"))
+
+	convertFilePathHashIndexToPrimaryKey(mg)
+	convertFileMetaPathHashKeyIndexToPrimaryKey(mg)
+}
+
+// This converts the existing unique constraint UQE_file_path_hash to a primary key in file table
+func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) {
+	// migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON
+	// in this case we need to do everything in one sql statement
+	mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(`
+	  ALTER TABLE file
+	  DROP PRIMARY KEY,
+	  DROP COLUMN my_row_id,
+	  DROP INDEX UQE_file_path_hash,
+	  ADD PRIMARY KEY (path_hash);
+	`)
+	mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file", ColumnName: "my_row_id"}
+	mg.AddMigration("drop my_row_id and add primary key to file table if my_row_id exists (auto-generated mysql column)", mysqlMigration1)
+
+	mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file DROP INDEX UQE_file_path_hash`)
+	mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file", IndexName: "UQE_file_path_hash"}
+	mg.AddMigration("drop file_path unique index from file table if it exists (mysql)", mysqlMigration2)
+
+	mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file ADD PRIMARY KEY (path_hash);`)
+	mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file"}
+	mg.AddMigration("add primary key to file table if it doesn't exist (mysql)", mysqlMigration3)
+
+	postgres := `
+		DO $$
+		BEGIN
+			-- Drop the unique constraint if it exists
+			DROP INDEX IF EXISTS "UQE_file_path_hash";
+
+			-- Add primary key if it doesn't already exist
+			IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN
+				ALTER TABLE file ADD PRIMARY KEY (path_hash);
+			END IF;
+		END $$;
+	`
+
+	sqlite := `
+		-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file" command after running migration.
+		CREATE TABLE file_new
+		(
+			path                    TEXT     NOT NULL,
+			path_hash               TEXT     NOT NULL,
+			parent_folder_path_hash TEXT     NOT NULL,
+			contents                BLOB     NOT NULL,
+			etag                    TEXT     NOT NULL,
+			cache_control           TEXT     NOT NULL,
+			content_disposition     TEXT     NOT NULL,
+			updated                 DATETIME NOT NULL,
+			created                 DATETIME NOT NULL,
+			size                    INTEGER  NOT NULL,
+			mime_type               TEXT     NOT NULL,
+
+			PRIMARY KEY (path_hash)
+		);
+
+		INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type)
+		SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file;
+
+		DROP TABLE file;
+		ALTER TABLE file_new RENAME TO file;
+
+		CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash);
+	`
+
+	// postgres and sqlite statements are idempotent so we can have only one condition-less migration
+	migration := migrator.NewRawSQLMigration("").
+		Postgres(postgres).
+		SQLite(sqlite)
+
+	mg.AddMigration("add primary key to file table (postgres and sqlite)", migration)
+}
+
+// This converts the existing unique constraint UQE_file_meta_path_hash_key to a primary key in file_meta table
+func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) {
+	// migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON
+	// in this case we need to do everything in one sql statement
+	mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(`
+	  ALTER TABLE file_meta
+      DROP PRIMARY KEY,
+      DROP COLUMN my_row_id,
+      DROP INDEX UQE_file_meta_path_hash_key,
+      ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
+	`)
+	mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file_meta", ColumnName: "my_row_id"}
+	mg.AddMigration("drop my_row_id and add primary key to file_meta table if my_row_id exists (auto-generated mysql column)", mysqlMigration1)
+
+	mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta DROP INDEX UQE_file_meta_path_hash_key`)
+	mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file_meta", IndexName: "UQE_file_meta_path_hash_key"}
+	mg.AddMigration("drop file_path unique index from file_meta table if it exists (mysql)", mysqlMigration2)
+
+	mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);`)
+	mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta"}
+	mg.AddMigration("add primary key to file_meta table if it doesn't exist (mysql)", mysqlMigration3)
+
+	postgres := `
+		DO $$
+		BEGIN
+			-- Drop the unique constraint if it exists
+			DROP INDEX IF EXISTS "UQE_file_meta_path_hash_key";
+
+			-- Add primary key if it doesn't already exist
+			IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file_meta'::regclass AND indisprimary) THEN
+				ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
+			END IF;
+		END $$;
+	`
+
+	sqlite := `
+		-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file_meta" command after running migration.
+		CREATE TABLE file_meta_new
+		(
+			path_hash TEXT NOT NULL,
+			key       TEXT NOT NULL,
+			value     TEXT NOT NULL,
+
+			PRIMARY KEY (path_hash, key)
+		);
+
+		INSERT INTO file_meta_new (path_hash, key, value)
+		SELECT path_hash, key, value FROM file_meta;
+
+		DROP TABLE file_meta;
+		ALTER TABLE file_meta_new RENAME TO file_meta;
+	`
+
+	// postgres and sqlite statements are idempotent so we can have only one condition-less migration
+	migration := migrator.NewRawSQLMigration("").
+		Postgres(postgres).
+		SQLite(sqlite)
+
+	mg.AddMigration("add primary key to file_meta table (postgres and sqlite)", migration)
 }
@@ -46,3 +46,28 @@ type IfColumnNotExistsCondition struct {
 func (c *IfColumnNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) {
 	return dialect.ColumnCheckSQL(c.TableName, c.ColumnName)
 }
+
+type IfColumnExistsCondition struct {
+	ExistsMigrationCondition
+	TableName  string
+	ColumnName string
+}
+
+func (c *IfColumnExistsCondition) SQL(dialect Dialect) (string, []interface{}) {
+	return dialect.ColumnCheckSQL(c.TableName, c.ColumnName)
+}
+
+type IfPrimaryKeyNotExistsCondition struct {
+	NotExistsMigrationCondition
+	TableName  string
+	ColumnName string
+}
+
+func (c *IfPrimaryKeyNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) {
+	// only use it with mysql
+	if dialect.DriverName() == "mysql" {
+		return "SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME=? AND COLUMN_KEY='PRI'", []interface{}{c.TableName}
+	}
+
+	return "", nil
+}
@@ -254,7 +254,7 @@ func (b *BaseDialect) CopyTableData(sourceTable string, targetTable string, sour
 	targetColsSQL := b.QuoteColList(targetCols)

 	quote := b.dialect.Quote
-	return fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable))
+	return fmt.Sprintf("INSERT INTO %s (%s)\nSELECT %s\nFROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable))
 }

 func (b *BaseDialect) DropTable(tableName string) string {
@@ -1,6 +1,8 @@
 package migrator

 import (
+	"fmt"
+	"slices"
 	"strings"
 )

@@ -271,3 +273,155 @@ func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharset
 func (m *TableCharsetMigration) SQL(d Dialect) string {
 	return d.UpdateTableSQL(m.tableName, m.columns)
 }
+
+type addPrimaryKeyMigration struct {
+	MigrationBase
+	tableName string
+	uniqueKey Index
+
+	// Used for Sqlite recreation of the table. Temporary table will have tableName + "_new" suffix.
+	table Table
+}
+
+func (m *addPrimaryKeyMigration) SQL(d Dialect) string {
+	if d.DriverName() == SQLite {
+		// Final SQL will do following in the individual statements:
+		// 1. Create new temporary table
+		// 2. Copy data from old table to temporary table
+		// 3. Drop old table, rename temporary table to original name
+		// 4. Recreate indexes for table.
+		//
+		// For example:
+		//
+		//	CREATE TABLE file_new
+		//	(
+		//		path                    TEXT     NOT NULL,
+		//		path_hash               TEXT     NOT NULL,
+		//		parent_folder_path_hash TEXT     NOT NULL,
+		//		contents                BLOB     NOT NULL,
+		//		etag                    TEXT     NOT NULL,
+		//		cache_control           TEXT     NOT NULL,
+		//		content_disposition     TEXT     NOT NULL,
+		//		updated                 DATETIME NOT NULL,
+		//		created                 DATETIME NOT NULL,
+		//		size                    INTEGER  NOT NULL,
+		//		mime_type               TEXT     NOT NULL,
+		//
+		//		PRIMARY KEY (path_hash)
+		//	);
+		//
+		//	INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type)
+		//	SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file;
+		//
+		//	DROP TABLE file;
+		//	ALTER TABLE file_new RENAME TO file;
+		//
+		//	CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash);
+
+		tempTable := m.table
+		tempTable.Name = m.tableName + "_new"
+
+		statements := strings.Builder{}
+
+		statements.WriteString(d.CreateTableSQL(&tempTable))
+		statements.WriteString("\n") // CreateTableSQL adds semicolon
+
+		cols := make([]string, 0, len(tempTable.Columns))
+		for _, col := range tempTable.Columns {
+			cols = append(cols, col.Name)
+		}
+		statements.WriteString(d.CopyTableData(m.tableName, tempTable.Name, cols, cols))
+		statements.WriteString(";\n")
+
+		statements.WriteString(d.DropTable(m.tableName))
+		statements.WriteString(";\n")
+
+		statements.WriteString(d.RenameTable(tempTable.Name, m.tableName))
+		statements.WriteString(";\n")
+
+		for _, idx := range tempTable.Indices {
+			// Use real table name, not temporary one now
+			statements.WriteString(d.CreateIndexSQL(m.tableName, idx))
+			statements.WriteString("\n") // CreateIndexSQL adds semicolon
+		}
+
+		return statements.String()
+	} else if d.DriverName() == Postgres {
+		quotesCols := make([]string, 0, len(m.uniqueKey.Cols))
+		for _, c := range m.uniqueKey.Cols {
+			quotesCols = append(quotesCols, d.Quote(c))
+		}
+
+		return fmt.Sprintf(`
+		DO $$
+		BEGIN
+			-- Drop the unique constraint if it exists
+			DROP INDEX IF EXISTS %s;
+
+			-- Add primary key if it doesn't already exist
+			IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = '%s'::regclass AND indisprimary) THEN
+				ALTER TABLE %s ADD PRIMARY KEY (%s);
+			END IF;
+		END $$;`, d.Quote(m.uniqueKey.XName(m.tableName)), m.tableName, d.Quote(m.tableName), strings.Join(quotesCols, ","))
+	} else {
+		return ""
+	}
+}
+
+// ConvertUniqueKeyToPrimaryKey adds series of migrations to convert existing unique key to PRIMARY KEY.
+// For Sqlite this means recreating the table, which only works if there are no foreign keys referencing the table.
+func ConvertUniqueKeyToPrimaryKey(mg *Migrator, uniqueKey Index, finalTable Table) {
+	tableName := finalTable.Name
+	if tableName == "" {
+		panic("invalid table name")
+	}
+	if len(uniqueKey.Cols) == 0 || uniqueKey.Type != UniqueIndex {
+		panic("invalid unique type")
+	}
+	if !slices.Equal(uniqueKey.Cols, finalTable.PrimaryKeys) {
+		panic("invalid primary key in the final table")
+	}
+
+	colPks := map[string]bool{}
+	for _, col := range finalTable.Columns {
+		if col.IsPrimaryKey {
+			colPks[col.Name] = true
+		}
+	}
+	for _, c := range uniqueKey.Cols {
+		if !colPks[c] {
+			panic(fmt.Sprintf("column %s is not part of primary key in the table definition", c))
+		}
+	}
+
+	columnsList := strings.Join(uniqueKey.Cols, ",")
+
+	mysqlQuote := NewDialect(MySQL).Quote
+	mysqlQuotedColumns := make([]string, 0, len(uniqueKey.Cols))
+	for _, col := range uniqueKey.Cols {
+		mysqlQuotedColumns = append(mysqlQuotedColumns, mysqlQuote(col))
+	}
+
+	// migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON
+	// in this case we need to do the conversion in one sql statement
+	mysqlMigration1 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`
+	  ALTER TABLE %s
+	  DROP PRIMARY KEY,
+	  DROP COLUMN my_row_id,
+	  DROP INDEX %s,
+	  ADD PRIMARY KEY (%s);
+	`, tableName, uniqueKey.XName(tableName), strings.Join(mysqlQuotedColumns, ",")))
+	mysqlMigration1.Condition = &IfColumnExistsCondition{TableName: tableName, ColumnName: "my_row_id"}
+	mg.AddMigration(fmt.Sprintf("drop my_row_id and add primary key with columns %s to table %s if my_row_id exists (auto-generated mysql column)", columnsList, tableName), mysqlMigration1)
+
+	mysqlMigration2 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s DROP INDEX %s`, tableName, uniqueKey.XName(tableName)))
+	mysqlMigration2.Condition = &IfIndexExistsCondition{TableName: tableName, IndexName: uniqueKey.XName(tableName)}
+	mg.AddMigration(fmt.Sprintf("drop unique index %s from %s table if it exists (mysql)", uniqueKey.XName(tableName), tableName), mysqlMigration2)
+
+	mysqlMigration3 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s ADD PRIMARY KEY (%s)`, tableName, strings.Join(mysqlQuotedColumns, ",")))
+	mysqlMigration3.Condition = &IfPrimaryKeyNotExistsCondition{TableName: tableName}
+	mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s if it doesn't exist (mysql)", columnsList, tableName), mysqlMigration3)
+
+	// postgres and sqlite statements are idempotent so we can have only one condition-less migration
+	mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s (postgres and sqlite)", columnsList, tableName), &addPrimaryKeyMigration{tableName: tableName, uniqueKey: uniqueKey, table: finalTable})
+}
@@ -0,0 +1,79 @@
+package migrator
+
+import (
+	_ "embed"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed testdata/sqlite_file_migration_statement.sql
+var sqliteMigrationStatement string
+
+func TestConvertUniqueKeyToPrimaryKey(t *testing.T) {
+	names := []string{
+		"drop my_row_id and add primary key with columns path_hash,etag to table file if my_row_id exists (auto-generated mysql column)",
+		"drop unique index UQE_file_path_hash_etag from file table if it exists (mysql)",
+		"add primary key with columns path_hash,etag to table file if it doesn't exist (mysql)",
+		"add primary key with columns path_hash,etag to table file (postgres and sqlite)",
+	}
+	expectedMigrations := map[string][]ExpectedMigration{
+		MySQL: {
+			{Id: names[0], SQL: `
+			  ALTER TABLE file
+			  DROP PRIMARY KEY,
+			  DROP COLUMN my_row_id,
+			  DROP INDEX UQE_file_path_hash_etag,
+			  ADD PRIMARY KEY (` + "`path_hash`" + `,` + "`etag`" + `);`},
+			{Id: names[1], SQL: "ALTER TABLE file DROP INDEX UQE_file_path_hash_etag"},
+			{Id: names[2], SQL: "ALTER TABLE file ADD PRIMARY KEY (`path_hash`,`etag`)"},
+			{Id: names[3], SQL: ""},
+		},
+		Postgres: {
+			{Id: names[0], SQL: ""},
+			{Id: names[1], SQL: ""},
+			{Id: names[2], SQL: ""},
+			{Id: names[3], SQL: `
+				DO $$
+				BEGIN
+					-- Drop the unique constraint if it exists
+					DROP INDEX IF EXISTS "UQE_file_path_hash_etag";
+
+					-- Add primary key if it doesn't already exist
+					IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN
+					ALTER TABLE "file" ADD PRIMARY KEY ("path_hash","etag");
+				END IF;
+				END $$;`},
+		},
+		SQLite: {
+			{Id: names[0], SQL: ""},
+			{Id: names[1], SQL: ""},
+			{Id: names[2], SQL: ""},
+			{Id: names[3], SQL: sqliteMigrationStatement}, // Embed used here because sqlite statement is full of backquotes.
+		},
+	}
+
+	for dialectName, migrations := range expectedMigrations {
+		t.Run(dialectName, func(t *testing.T) {
+			err := CheckExpectedMigrations(dialectName, migrations, func(migrator *Migrator) {
+				ConvertUniqueKeyToPrimaryKey(migrator,
+					Index{Cols: []string{"path_hash", "etag"}, Type: UniqueIndex}, // Convert this unique key to primary key
+					Table{
+						Name: "file",
+						Columns: []*Column{
+							{Name: "path", Type: DB_NVarchar, Length: 1024, Nullable: false},
+							{Name: "path_hash", Type: DB_NVarchar, Length: 64, Nullable: false, IsPrimaryKey: true},
+							{Name: "parent_folder_path_hash", Type: DB_NVarchar, Length: 64, Nullable: false},
+							{Name: "contents", Type: DB_Blob, Nullable: false},
+							{Name: "etag", Type: DB_NVarchar, Length: 32, Nullable: false, IsPrimaryKey: true},
+						},
+						PrimaryKeys: []string{"path_hash", "etag"},
+						Indices: []*Index{
+							{Cols: []string{"parent_folder_path_hash"}},
+						},
+					})
+			})
+			require.NoError(t, err)
+		})
+	}
+}
@@ -68,12 +68,16 @@ func NewMigrator(engine *xorm.Engine, cfg *setting.Cfg) *Migrator {

 // NewScopedMigrator should only be used for the transition to a new storage engine
 func NewScopedMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string) *Migrator {
+	return newMigrator(engine, cfg, scope, NewDialect(engine.DriverName()))
+}
+
+func newMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string, dialect Dialect) *Migrator {
 	mg := &Migrator{
 		Cfg:          cfg,
 		DBEngine:     engine,
 		migrations:   make([]Migration, 0),
 		migrationIds: make(map[string]struct{}),
-		Dialect:      NewDialect(engine.DriverName()),
+		Dialect:      dialect,
 		metrics: migratorMetrics{
 			migCount: prometheus.NewCounterVec(prometheus.CounterOpts{
 				Namespace: "grafana_database",
@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS `file_new` (
+    `path` TEXT NOT NULL
+    , `path_hash` TEXT NOT NULL
+    , `parent_folder_path_hash` TEXT NOT NULL
+    , `contents` BLOB NOT NULL
+    , `etag` TEXT NOT NULL
+    , PRIMARY KEY ( `path_hash`,`etag` ));
+
+INSERT INTO `file_new` (`path`
+  , `path_hash`
+  , `parent_folder_path_hash`
+  , `contents`
+  , `etag`)
+SELECT `path`
+  , `path_hash`
+  , `parent_folder_path_hash`
+  , `contents`
+  , `etag`
+FROM `file`;
+
+DROP TABLE IF EXISTS `file`;
+ALTER TABLE `file_new` RENAME TO `file`;
+CREATE INDEX `IDX_file_parent_folder_path_hash` ON `file` (`parent_folder_path_hash`);
@@ -0,0 +1,51 @@
+package migrator
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ExpectedMigration struct {
+	Id  string
+	SQL string
+}
+
+// CheckExpectedMigrations verifies that given migrations exist in migrator after running addMigrations function,
+// that they are in the same order and have expected SQL.
+func CheckExpectedMigrations(dialectName string, expected []ExpectedMigration, addMigrations func(migrator *Migrator)) error {
+	d := NewDialect(dialectName)
+	mg := newMigrator(nil, nil, "", d)
+	addMigrations(mg)
+
+	migrations := mg.migrations
+	migrationNames := make([]string, 0, len(migrations))
+	for _, m := range expected {
+		for ; len(migrations) > 0 && migrations[0].Id() != m.Id; migrations = migrations[1:] {
+			migrationNames = append(migrationNames, migrations[0].Id())
+		}
+
+		if len(migrations) == 0 {
+			return fmt.Errorf("migration `%s` not found, existing migrations:\n%s", m.Id, strings.Join(migrationNames, "\n"))
+		}
+
+		sql := migrations[0].SQL(d)
+		if normalizeLines(m.SQL) != normalizeLines(sql) {
+			return fmt.Errorf("migration `%s` has wrong SQL:\nexpected:\n%s\nactual:\n%s", m.Id, m.SQL, sql)
+		}
+	}
+	return nil
+}
+
+func normalizeLines(sql string) string {
+	lines := strings.Split(sql, "\n")
+	result := strings.Builder{}
+	for _, l := range lines {
+		l := strings.TrimSpace(l)
+		if l == "" {
+			continue
+		}
+		result.WriteString(l)
+		result.WriteString("\n")
+	}
+	return result.String()
+}
@@ -4,15 +4,18 @@ import (
 	"context"
 	"testing"

+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/storage/secret/database"
 	"github.com/grafana/grafana/pkg/storage/secret/migrator"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/otel/trace/noop"
 )

 func TestEncryptedValueStoreImpl(t *testing.T) {
+	t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2")
+
 	// Initialize data key storage with a fake db
 	testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
 	tracer := noop.NewTracerProvider().Tracer("test")
@@ -45,7 +45,7 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {

 	tables := []migrator.Table{}

-	tables = append(tables, migrator.Table{
+	secureValueTable := migrator.Table{
 		Name: TableNameSecureValue,
 		Columns: []*migrator.Column{
 			// Kubernetes Metadata
@@ -74,7 +74,8 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 			{Cols: []string{"namespace", "name", "version", "active"}, Type: migrator.UniqueIndex},
 			{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex},
 		},
-	})
+	}
+	tables = append(tables, secureValueTable)

 	tables = append(tables, migrator.Table{
 		Name: TableNameKeeper,
@@ -101,34 +102,37 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 		},
 	})

-	// TODO -- document how the seemingly arbitrary column lengths were chosen
-	// The answer for now is that they come from the legacy secrets service, but it would be good to know that they will still work in the new service
-	tables = append(tables, migrator.Table{
+	dataKeyTable := migrator.Table{
 		Name: TableNameDataKey,
 		Columns: []*migrator.Column{
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true},
+			{Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true},    // Arbitrarily chosen.
 			{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
-			{Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false},
+			{Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false}, // Arbitrarily chosen.
 			{Name: "active", Type: migrator.DB_Bool, Nullable: false},
-			{Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false},
+			{Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false}, // Arbitrarily chosen.
 			{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
 			{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
 			{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
 		},
-		Indices: []*migrator.Index{}, // TODO: add indexes based on the queries we make.
-	})
+		Indices: []*migrator.Index{},
+	}
+	tables = append(tables, dataKeyTable)

-	tables = append(tables, migrator.Table{
+	encryptedValueTable := migrator.Table{
 		Name: TableNameEncryptedValue,
 		Columns: []*migrator.Column{
 			{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true},     // Fixed size of a UUID.
+			{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false},
+			{Name: "version", Type: migrator.DB_BigInt, Nullable: false},
 			{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
 			{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
 			{Name: "updated", Type: migrator.DB_BigInt, Nullable: false},
 		},
-		Indices: []*migrator.Index{}, // TODO: add indexes based on the queries we make.
-	})
+		Indices: []*migrator.Index{
+			{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex},
+		},
+	}
+	tables = append(tables, encryptedValueTable)

 	// Initialize all tables
 	for t := range tables {
@@ -138,4 +142,82 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 			mg.AddMigration(fmt.Sprintf("create table %s, index: %d", tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i]))
 		}
 	}
+
+	mg.AddMigration("create index for list on "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"namespace", "active", "updated"},
+		Type: migrator.IndexType,
+	}))
+
+	mg.AddMigration("create index for list and read current on "+TableNameDataKey, migrator.NewAddIndexMigration(dataKeyTable, &migrator.Index{
+		Cols: []string{"namespace", "label", "active"},
+		Type: migrator.IndexType,
+	}))
+
+	// Owner Reference columns
+	mg.AddMigration("add owner_reference_api_group column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "owner_reference_api_group",
+		Type:     migrator.DB_NVarchar,
+		Length:   253, // Limit enforced by K8s.
+		Nullable: true,
+	}))
+
+	mg.AddMigration("add owner_reference_api_version column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "owner_reference_api_version",
+		Type:     migrator.DB_NVarchar,
+		Length:   253, // Limit enforced by K8s.
+		Nullable: true,
+	}))
+
+	mg.AddMigration("add owner_reference_kind column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "owner_reference_kind",
+		Type:     migrator.DB_NVarchar,
+		Length:   253, // Limit enforced by K8s.
+		Nullable: true,
+	}))
+
+	mg.AddMigration("add owner_reference_name column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "owner_reference_name",
+		Type:     migrator.DB_NVarchar,
+		Length:   253, // Limit enforced by K8s.
+		Nullable: true,
+	}))
+
+	mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_token",
+		Type:     migrator.DB_NVarchar,
+		Length:   36,
+		Nullable: true,
+	}))
+	mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_token"},
+	}))
+	mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_created",
+		Type:     migrator.DB_BigInt,
+		Nullable: false,
+		Default:  "0",
+	}))
+	mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_created"},
+	}))
+
+	encryptedValueTableUniqueKey := migrator.Index{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}
+	updatedEncryptedValueTable := migrator.Table{
+		Name: TableNameEncryptedValue,
+		Columns: []*migrator.Column{
+			{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, // Limit enforced by K8s.
+			{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true},
+			{Name: "version", Type: migrator.DB_BigInt, Nullable: false, IsPrimaryKey: true},
+			{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
+			{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
+			{Name: "updated", Type: migrator.DB_BigInt, Nullable: false},
+			// {Name: "data_key_id", Type: migrator.DB_NVarchar, Length: 100, Nullable: false, Default: "''"}, // TODO: Not present until Grafana 12.3.
+		},
+		PrimaryKeys: []string{"namespace", "name", "version"},
+		// TODO: Not present until Grafana 12.3
+		//Indices: []*migrator.Index{
+		//	{Cols: []string{"data_key_id"}},
+		//},
+	}
+	migrator.ConvertUniqueKeyToPrimaryKey(mg, encryptedValueTableUniqueKey, updatedEncryptedValueTable)
 }
@@ -167,5 +167,17 @@ func initResourceTables(mg *migrator.Migrator) string {
 		Name: "IDX_resource_history_namespace_group_resource_name_generation",
 	}))

+	oldResourceVersionUniqueKey := migrator.Index{Cols: []string{"group", "resource"}, Type: migrator.UniqueIndex}
+	updatedResourceVersionTable := migrator.Table{
+		Name: "resource_version",
+		Columns: []*migrator.Column{
+			{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true},
+			{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true},
+			{Name: "resource_version", Type: migrator.DB_BigInt, Nullable: false},
+		},
+		PrimaryKeys: []string{"group", "resource"},
+	}
+
+	migrator.ConvertUniqueKeyToPrimaryKey(mg, oldResourceVersionUniqueKey, updatedResourceVersionTable)
 	return marker
 }