diff --git a/pkg/registry/apis/secret/register.go b/pkg/registry/apis/secret/register.go index f43b164f6b4..d0877a4c598 100644 --- a/pkg/registry/apis/secret/register.go +++ b/pkg/registry/apis/secret/register.go @@ -19,6 +19,14 @@ func RegisterDependencies( secretDBMigrator contracts.SecretDBMigrator, accessControlService accesscontrol.Service, ) (*DependencyRegisterer, error) { + // Some DBs that claim to be MySQL/Postgres-compatible might not support table locking. + lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true) + + // This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator. + if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil { + return nil, fmt.Errorf("running secret database migrations: %w", err) + } + if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) || !features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) { return nil, nil } @@ -28,16 +36,5 @@ func RegisterDependencies( return nil, fmt.Errorf("registering access control roles: %w", err) } - // We shouldn't need to create the DB in HG, as that will use the MT api server. - if cfg.StackID == "" { - // Some DBs that claim to be MySQL/Postgres-compatible might not support table locking. - lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true) - - // This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator. - if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil { - return nil, fmt.Errorf("running secret database migrations: %w", err) - } - } - return &DependencyRegisterer{}, nil } diff --git a/pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go b/pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go index fa9aa7a6ba5..ea80e9364f8 100644 --- a/pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go +++ b/pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go @@ -26,6 +26,8 @@ func TestMain(m *testing.M) { } func Test_SQLKeeperSetup(t *testing.T) { + t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2") + ctx := context.Background() namespace1 := "namespace1" namespace2 := "namespace2" diff --git a/pkg/registry/apis/secret/testutils/testutils.go b/pkg/registry/apis/secret/testutils/testutils.go index 281304320c8..939c02b4d0a 100644 --- a/pkg/registry/apis/secret/testutils/testutils.go +++ b/pkg/registry/apis/secret/testutils/testutils.go @@ -4,11 +4,14 @@ import ( "context" "testing" - secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1" - encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption" "go.opentelemetry.io/otel/trace/noop" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1" + encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption" + + "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" "github.com/grafana/grafana/pkg/registry/apis/secret/decrypt" @@ -25,7 +28,6 @@ import ( "github.com/grafana/grafana/pkg/storage/secret/database" "github.com/grafana/grafana/pkg/storage/secret/metadata" "github.com/grafana/grafana/pkg/storage/secret/migrator" - "github.com/stretchr/testify/require" ) type SetupConfig struct { @@ -50,6 +52,8 @@ func WithMutateCfg(f func(*SetupConfig)) func(*SetupConfig) { } func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut { + t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2") + setupCfg := defaultSetupCfg() for _, opt := range opts { opt(&setupCfg) diff --git a/pkg/services/sqlstore/migrations/cloud_migrations.go b/pkg/services/sqlstore/migrations/cloud_migrations.go index 18dc3590570..1159a28da32 100644 --- a/pkg/services/sqlstore/migrations/cloud_migrations.go +++ b/pkg/services/sqlstore/migrations/cloud_migrations.go @@ -187,4 +187,33 @@ func addCloudMigrationsMigrations(mg *Migrator) { mg.AddMigration("increase resource_uid column length", NewRawSQLMigration(""). Mysql("ALTER TABLE cloud_migration_resource MODIFY resource_uid NVARCHAR(255);"). Postgres("ALTER TABLE cloud_migration_resource ALTER COLUMN resource_uid TYPE VARCHAR(255);")) + + migrationSnapshotPartitionTable := Table{ + Name: "cloud_migration_snapshot_partition", + Columns: []*Column{ + {Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false}, + {Name: "partition_number", Type: DB_Int, Nullable: false}, + {Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false}, + {Name: "data", Type: DB_LongBlob, Nullable: false}, + }, + } + + mg.AddMigration("create cloud_migration_snapshot_partition table v1", NewAddTableMigration(migrationSnapshotPartitionTable)) + srpUniqueIndex := Index{ + Name: "srp_unique", + Cols: []string{"snapshot_uid", "resource_type", "partition_number"}, Type: UniqueIndex, + } + mg.AddMigration("add cloud_migration_snapshot_partition srp_unique index", NewAddIndexMigration(migrationSnapshotPartitionTable, &srpUniqueIndex)) + + updatedCloudMigrationSnapshotPartitionTable := Table{ + Name: "cloud_migration_snapshot_partition", + Columns: []*Column{ + {Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true}, + {Name: "partition_number", Type: DB_Int, Nullable: false, IsPrimaryKey: true}, + {Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false, IsPrimaryKey: true}, + {Name: "data", Type: DB_LongBlob, Nullable: false}, + }, + PrimaryKeys: []string{"snapshot_uid", "resource_type", "partition_number"}, + } + ConvertUniqueKeyToPrimaryKey(mg, srpUniqueIndex, updatedCloudMigrationSnapshotPartitionTable) } diff --git a/pkg/services/sqlstore/migrations/db_file_storage.go b/pkg/services/sqlstore/migrations/db_file_storage.go index fcc49ae0d57..c44344307ac 100644 --- a/pkg/services/sqlstore/migrations/db_file_storage.go +++ b/pkg/services/sqlstore/migrations/db_file_storage.go @@ -67,4 +67,139 @@ func addDbFileStorageMigration(mg *migrator.Migrator) { mg.AddMigration("migrate contents column to mediumblob for MySQL", migrator.NewRawSQLMigration(""). Mysql("ALTER TABLE file MODIFY contents MEDIUMBLOB;")) + + convertFilePathHashIndexToPrimaryKey(mg) + convertFileMetaPathHashKeyIndexToPrimaryKey(mg) +} + +// This converts the existing unique constraint UQE_file_path_hash to a primary key in file table +func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) { + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do everything in one sql statement + mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(` + ALTER TABLE file + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_path_hash, + ADD PRIMARY KEY (path_hash); + `) + mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file", ColumnName: "my_row_id"} + mg.AddMigration("drop my_row_id and add primary key to file table if my_row_id exists (auto-generated mysql column)", mysqlMigration1) + + mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file DROP INDEX UQE_file_path_hash`) + mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file", IndexName: "UQE_file_path_hash"} + mg.AddMigration("drop file_path unique index from file table if it exists (mysql)", mysqlMigration2) + + mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file ADD PRIMARY KEY (path_hash);`) + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file"} + mg.AddMigration("add primary key to file table if it doesn't exist (mysql)", mysqlMigration3) + + postgres := ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_path_hash"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN + ALTER TABLE file ADD PRIMARY KEY (path_hash); + END IF; + END $$; + ` + + sqlite := ` + -- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file" command after running migration. + CREATE TABLE file_new + ( + path TEXT NOT NULL, + path_hash TEXT NOT NULL, + parent_folder_path_hash TEXT NOT NULL, + contents BLOB NOT NULL, + etag TEXT NOT NULL, + cache_control TEXT NOT NULL, + content_disposition TEXT NOT NULL, + updated DATETIME NOT NULL, + created DATETIME NOT NULL, + size INTEGER NOT NULL, + mime_type TEXT NOT NULL, + + PRIMARY KEY (path_hash) + ); + + INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type) + SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file; + + DROP TABLE file; + ALTER TABLE file_new RENAME TO file; + + CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash); + ` + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + migration := migrator.NewRawSQLMigration(""). + Postgres(postgres). + SQLite(sqlite) + + mg.AddMigration("add primary key to file table (postgres and sqlite)", migration) +} + +// This converts the existing unique constraint UQE_file_meta_path_hash_key to a primary key in file_meta table +func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) { + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do everything in one sql statement + mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(` + ALTER TABLE file_meta + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_meta_path_hash_key, + ADD PRIMARY KEY (path_hash, ` + "`key`" + `); + `) + mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file_meta", ColumnName: "my_row_id"} + mg.AddMigration("drop my_row_id and add primary key to file_meta table if my_row_id exists (auto-generated mysql column)", mysqlMigration1) + + mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta DROP INDEX UQE_file_meta_path_hash_key`) + mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file_meta", IndexName: "UQE_file_meta_path_hash_key"} + mg.AddMigration("drop file_path unique index from file_meta table if it exists (mysql)", mysqlMigration2) + + mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);`) + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta"} + mg.AddMigration("add primary key to file_meta table if it doesn't exist (mysql)", mysqlMigration3) + + postgres := ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_meta_path_hash_key"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file_meta'::regclass AND indisprimary) THEN + ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `); + END IF; + END $$; + ` + + sqlite := ` + -- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file_meta" command after running migration. + CREATE TABLE file_meta_new + ( + path_hash TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + + PRIMARY KEY (path_hash, key) + ); + + INSERT INTO file_meta_new (path_hash, key, value) + SELECT path_hash, key, value FROM file_meta; + + DROP TABLE file_meta; + ALTER TABLE file_meta_new RENAME TO file_meta; + ` + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + migration := migrator.NewRawSQLMigration(""). + Postgres(postgres). + SQLite(sqlite) + + mg.AddMigration("add primary key to file_meta table (postgres and sqlite)", migration) } diff --git a/pkg/services/sqlstore/migrator/conditions.go b/pkg/services/sqlstore/migrator/conditions.go index 4fc7c780e84..b751be61c0e 100644 --- a/pkg/services/sqlstore/migrator/conditions.go +++ b/pkg/services/sqlstore/migrator/conditions.go @@ -46,3 +46,28 @@ type IfColumnNotExistsCondition struct { func (c *IfColumnNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) { return dialect.ColumnCheckSQL(c.TableName, c.ColumnName) } + +type IfColumnExistsCondition struct { + ExistsMigrationCondition + TableName string + ColumnName string +} + +func (c *IfColumnExistsCondition) SQL(dialect Dialect) (string, []interface{}) { + return dialect.ColumnCheckSQL(c.TableName, c.ColumnName) +} + +type IfPrimaryKeyNotExistsCondition struct { + NotExistsMigrationCondition + TableName string + ColumnName string +} + +func (c *IfPrimaryKeyNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) { + // only use it with mysql + if dialect.DriverName() == "mysql" { + return "SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME=? AND COLUMN_KEY='PRI'", []interface{}{c.TableName} + } + + return "", nil +} diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index a4d5c3061dd..ba4e451e601 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -254,7 +254,7 @@ func (b *BaseDialect) CopyTableData(sourceTable string, targetTable string, sour targetColsSQL := b.QuoteColList(targetCols) quote := b.dialect.Quote - return fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable)) + return fmt.Sprintf("INSERT INTO %s (%s)\nSELECT %s\nFROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable)) } func (b *BaseDialect) DropTable(tableName string) string { diff --git a/pkg/services/sqlstore/migrator/migrations.go b/pkg/services/sqlstore/migrator/migrations.go index b785fd867d5..ebc9573d111 100644 --- a/pkg/services/sqlstore/migrator/migrations.go +++ b/pkg/services/sqlstore/migrator/migrations.go @@ -1,6 +1,8 @@ package migrator import ( + "fmt" + "slices" "strings" ) @@ -271,3 +273,155 @@ func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharset func (m *TableCharsetMigration) SQL(d Dialect) string { return d.UpdateTableSQL(m.tableName, m.columns) } + +type addPrimaryKeyMigration struct { + MigrationBase + tableName string + uniqueKey Index + + // Used for Sqlite recreation of the table. Temporary table will have tableName + "_new" suffix. + table Table +} + +func (m *addPrimaryKeyMigration) SQL(d Dialect) string { + if d.DriverName() == SQLite { + // Final SQL will do following in the individual statements: + // 1. Create new temporary table + // 2. Copy data from old table to temporary table + // 3. Drop old table, rename temporary table to original name + // 4. Recreate indexes for table. + // + // For example: + // + // CREATE TABLE file_new + // ( + // path TEXT NOT NULL, + // path_hash TEXT NOT NULL, + // parent_folder_path_hash TEXT NOT NULL, + // contents BLOB NOT NULL, + // etag TEXT NOT NULL, + // cache_control TEXT NOT NULL, + // content_disposition TEXT NOT NULL, + // updated DATETIME NOT NULL, + // created DATETIME NOT NULL, + // size INTEGER NOT NULL, + // mime_type TEXT NOT NULL, + // + // PRIMARY KEY (path_hash) + // ); + // + // INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type) + // SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file; + // + // DROP TABLE file; + // ALTER TABLE file_new RENAME TO file; + // + // CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash); + + tempTable := m.table + tempTable.Name = m.tableName + "_new" + + statements := strings.Builder{} + + statements.WriteString(d.CreateTableSQL(&tempTable)) + statements.WriteString("\n") // CreateTableSQL adds semicolon + + cols := make([]string, 0, len(tempTable.Columns)) + for _, col := range tempTable.Columns { + cols = append(cols, col.Name) + } + statements.WriteString(d.CopyTableData(m.tableName, tempTable.Name, cols, cols)) + statements.WriteString(";\n") + + statements.WriteString(d.DropTable(m.tableName)) + statements.WriteString(";\n") + + statements.WriteString(d.RenameTable(tempTable.Name, m.tableName)) + statements.WriteString(";\n") + + for _, idx := range tempTable.Indices { + // Use real table name, not temporary one now + statements.WriteString(d.CreateIndexSQL(m.tableName, idx)) + statements.WriteString("\n") // CreateIndexSQL adds semicolon + } + + return statements.String() + } else if d.DriverName() == Postgres { + quotesCols := make([]string, 0, len(m.uniqueKey.Cols)) + for _, c := range m.uniqueKey.Cols { + quotesCols = append(quotesCols, d.Quote(c)) + } + + return fmt.Sprintf(` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS %s; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = '%s'::regclass AND indisprimary) THEN + ALTER TABLE %s ADD PRIMARY KEY (%s); + END IF; + END $$;`, d.Quote(m.uniqueKey.XName(m.tableName)), m.tableName, d.Quote(m.tableName), strings.Join(quotesCols, ",")) + } else { + return "" + } +} + +// ConvertUniqueKeyToPrimaryKey adds series of migrations to convert existing unique key to PRIMARY KEY. +// For Sqlite this means recreating the table, which only works if there are no foreign keys referencing the table. +func ConvertUniqueKeyToPrimaryKey(mg *Migrator, uniqueKey Index, finalTable Table) { + tableName := finalTable.Name + if tableName == "" { + panic("invalid table name") + } + if len(uniqueKey.Cols) == 0 || uniqueKey.Type != UniqueIndex { + panic("invalid unique type") + } + if !slices.Equal(uniqueKey.Cols, finalTable.PrimaryKeys) { + panic("invalid primary key in the final table") + } + + colPks := map[string]bool{} + for _, col := range finalTable.Columns { + if col.IsPrimaryKey { + colPks[col.Name] = true + } + } + for _, c := range uniqueKey.Cols { + if !colPks[c] { + panic(fmt.Sprintf("column %s is not part of primary key in the table definition", c)) + } + } + + columnsList := strings.Join(uniqueKey.Cols, ",") + + mysqlQuote := NewDialect(MySQL).Quote + mysqlQuotedColumns := make([]string, 0, len(uniqueKey.Cols)) + for _, col := range uniqueKey.Cols { + mysqlQuotedColumns = append(mysqlQuotedColumns, mysqlQuote(col)) + } + + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do the conversion in one sql statement + mysqlMigration1 := NewRawSQLMigration("").Mysql(fmt.Sprintf(` + ALTER TABLE %s + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX %s, + ADD PRIMARY KEY (%s); + `, tableName, uniqueKey.XName(tableName), strings.Join(mysqlQuotedColumns, ","))) + mysqlMigration1.Condition = &IfColumnExistsCondition{TableName: tableName, ColumnName: "my_row_id"} + mg.AddMigration(fmt.Sprintf("drop my_row_id and add primary key with columns %s to table %s if my_row_id exists (auto-generated mysql column)", columnsList, tableName), mysqlMigration1) + + mysqlMigration2 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s DROP INDEX %s`, tableName, uniqueKey.XName(tableName))) + mysqlMigration2.Condition = &IfIndexExistsCondition{TableName: tableName, IndexName: uniqueKey.XName(tableName)} + mg.AddMigration(fmt.Sprintf("drop unique index %s from %s table if it exists (mysql)", uniqueKey.XName(tableName), tableName), mysqlMigration2) + + mysqlMigration3 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s ADD PRIMARY KEY (%s)`, tableName, strings.Join(mysqlQuotedColumns, ","))) + mysqlMigration3.Condition = &IfPrimaryKeyNotExistsCondition{TableName: tableName} + mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s if it doesn't exist (mysql)", columnsList, tableName), mysqlMigration3) + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s (postgres and sqlite)", columnsList, tableName), &addPrimaryKeyMigration{tableName: tableName, uniqueKey: uniqueKey, table: finalTable}) +} diff --git a/pkg/services/sqlstore/migrator/migrations_test.go b/pkg/services/sqlstore/migrator/migrations_test.go new file mode 100644 index 00000000000..ebf7c4f48f6 --- /dev/null +++ b/pkg/services/sqlstore/migrator/migrations_test.go @@ -0,0 +1,79 @@ +package migrator + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/require" +) + +//go:embed testdata/sqlite_file_migration_statement.sql +var sqliteMigrationStatement string + +func TestConvertUniqueKeyToPrimaryKey(t *testing.T) { + names := []string{ + "drop my_row_id and add primary key with columns path_hash,etag to table file if my_row_id exists (auto-generated mysql column)", + "drop unique index UQE_file_path_hash_etag from file table if it exists (mysql)", + "add primary key with columns path_hash,etag to table file if it doesn't exist (mysql)", + "add primary key with columns path_hash,etag to table file (postgres and sqlite)", + } + expectedMigrations := map[string][]ExpectedMigration{ + MySQL: { + {Id: names[0], SQL: ` + ALTER TABLE file + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_path_hash_etag, + ADD PRIMARY KEY (` + "`path_hash`" + `,` + "`etag`" + `);`}, + {Id: names[1], SQL: "ALTER TABLE file DROP INDEX UQE_file_path_hash_etag"}, + {Id: names[2], SQL: "ALTER TABLE file ADD PRIMARY KEY (`path_hash`,`etag`)"}, + {Id: names[3], SQL: ""}, + }, + Postgres: { + {Id: names[0], SQL: ""}, + {Id: names[1], SQL: ""}, + {Id: names[2], SQL: ""}, + {Id: names[3], SQL: ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_path_hash_etag"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN + ALTER TABLE "file" ADD PRIMARY KEY ("path_hash","etag"); + END IF; + END $$;`}, + }, + SQLite: { + {Id: names[0], SQL: ""}, + {Id: names[1], SQL: ""}, + {Id: names[2], SQL: ""}, + {Id: names[3], SQL: sqliteMigrationStatement}, // Embed used here because sqlite statement is full of backquotes. + }, + } + + for dialectName, migrations := range expectedMigrations { + t.Run(dialectName, func(t *testing.T) { + err := CheckExpectedMigrations(dialectName, migrations, func(migrator *Migrator) { + ConvertUniqueKeyToPrimaryKey(migrator, + Index{Cols: []string{"path_hash", "etag"}, Type: UniqueIndex}, // Convert this unique key to primary key + Table{ + Name: "file", + Columns: []*Column{ + {Name: "path", Type: DB_NVarchar, Length: 1024, Nullable: false}, + {Name: "path_hash", Type: DB_NVarchar, Length: 64, Nullable: false, IsPrimaryKey: true}, + {Name: "parent_folder_path_hash", Type: DB_NVarchar, Length: 64, Nullable: false}, + {Name: "contents", Type: DB_Blob, Nullable: false}, + {Name: "etag", Type: DB_NVarchar, Length: 32, Nullable: false, IsPrimaryKey: true}, + }, + PrimaryKeys: []string{"path_hash", "etag"}, + Indices: []*Index{ + {Cols: []string{"parent_folder_path_hash"}}, + }, + }) + }) + require.NoError(t, err) + }) + } +} diff --git a/pkg/services/sqlstore/migrator/migrator.go b/pkg/services/sqlstore/migrator/migrator.go index d8e55c4ee71..76c2657bc47 100644 --- a/pkg/services/sqlstore/migrator/migrator.go +++ b/pkg/services/sqlstore/migrator/migrator.go @@ -68,12 +68,16 @@ func NewMigrator(engine *xorm.Engine, cfg *setting.Cfg) *Migrator { // NewScopedMigrator should only be used for the transition to a new storage engine func NewScopedMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string) *Migrator { + return newMigrator(engine, cfg, scope, NewDialect(engine.DriverName())) +} + +func newMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string, dialect Dialect) *Migrator { mg := &Migrator{ Cfg: cfg, DBEngine: engine, migrations: make([]Migration, 0), migrationIds: make(map[string]struct{}), - Dialect: NewDialect(engine.DriverName()), + Dialect: dialect, metrics: migratorMetrics{ migCount: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "grafana_database", diff --git a/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql b/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql new file mode 100644 index 00000000000..2f37839d842 --- /dev/null +++ b/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS `file_new` ( + `path` TEXT NOT NULL + , `path_hash` TEXT NOT NULL + , `parent_folder_path_hash` TEXT NOT NULL + , `contents` BLOB NOT NULL + , `etag` TEXT NOT NULL + , PRIMARY KEY ( `path_hash`,`etag` )); + +INSERT INTO `file_new` (`path` + , `path_hash` + , `parent_folder_path_hash` + , `contents` + , `etag`) +SELECT `path` + , `path_hash` + , `parent_folder_path_hash` + , `contents` + , `etag` +FROM `file`; + +DROP TABLE IF EXISTS `file`; +ALTER TABLE `file_new` RENAME TO `file`; +CREATE INDEX `IDX_file_parent_folder_path_hash` ON `file` (`parent_folder_path_hash`); diff --git a/pkg/services/sqlstore/migrator/testing.go b/pkg/services/sqlstore/migrator/testing.go new file mode 100644 index 00000000000..cfe7849c79e --- /dev/null +++ b/pkg/services/sqlstore/migrator/testing.go @@ -0,0 +1,51 @@ +package migrator + +import ( + "fmt" + "strings" +) + +type ExpectedMigration struct { + Id string + SQL string +} + +// CheckExpectedMigrations verifies that given migrations exist in migrator after running addMigrations function, +// that they are in the same order and have expected SQL. +func CheckExpectedMigrations(dialectName string, expected []ExpectedMigration, addMigrations func(migrator *Migrator)) error { + d := NewDialect(dialectName) + mg := newMigrator(nil, nil, "", d) + addMigrations(mg) + + migrations := mg.migrations + migrationNames := make([]string, 0, len(migrations)) + for _, m := range expected { + for ; len(migrations) > 0 && migrations[0].Id() != m.Id; migrations = migrations[1:] { + migrationNames = append(migrationNames, migrations[0].Id()) + } + + if len(migrations) == 0 { + return fmt.Errorf("migration `%s` not found, existing migrations:\n%s", m.Id, strings.Join(migrationNames, "\n")) + } + + sql := migrations[0].SQL(d) + if normalizeLines(m.SQL) != normalizeLines(sql) { + return fmt.Errorf("migration `%s` has wrong SQL:\nexpected:\n%s\nactual:\n%s", m.Id, m.SQL, sql) + } + } + return nil +} + +func normalizeLines(sql string) string { + lines := strings.Split(sql, "\n") + result := strings.Builder{} + for _, l := range lines { + l := strings.TrimSpace(l) + if l == "" { + continue + } + result.WriteString(l) + result.WriteString("\n") + } + return result.String() +} diff --git a/pkg/storage/secret/encryption/encrypted_value_store_test.go b/pkg/storage/secret/encryption/encrypted_value_store_test.go index 92afe58e7e2..5807eefd134 100644 --- a/pkg/storage/secret/encryption/encrypted_value_store_test.go +++ b/pkg/storage/secret/encryption/encrypted_value_store_test.go @@ -4,15 +4,18 @@ import ( "context" "testing" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/storage/secret/database" "github.com/grafana/grafana/pkg/storage/secret/migrator" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/trace/noop" ) func TestEncryptedValueStoreImpl(t *testing.T) { + t.Skip("feature FlagSecretsManagementAppPlatform is broken in 12.1.X for X >= 6 due to backporting incompatible DB schema, don't enable it before updating to 12.2") + // Initialize data key storage with a fake db testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New())) tracer := noop.NewTracerProvider().Tracer("test") diff --git a/pkg/storage/secret/migrator/migrator.go b/pkg/storage/secret/migrator/migrator.go index 6ee7209bc8a..cdfb11f88a5 100644 --- a/pkg/storage/secret/migrator/migrator.go +++ b/pkg/storage/secret/migrator/migrator.go @@ -45,7 +45,7 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) { tables := []migrator.Table{} - tables = append(tables, migrator.Table{ + secureValueTable := migrator.Table{ Name: TableNameSecureValue, Columns: []*migrator.Column{ // Kubernetes Metadata @@ -74,7 +74,8 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) { {Cols: []string{"namespace", "name", "version", "active"}, Type: migrator.UniqueIndex}, {Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}, }, - }) + } + tables = append(tables, secureValueTable) tables = append(tables, migrator.Table{ Name: TableNameKeeper, @@ -101,34 +102,37 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) { }, }) - // TODO -- document how the seemingly arbitrary column lengths were chosen - // The answer for now is that they come from the legacy secrets service, but it would be good to know that they will still work in the new service - tables = append(tables, migrator.Table{ + dataKeyTable := migrator.Table{ Name: TableNameDataKey, Columns: []*migrator.Column{ - {Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true}, + {Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true}, // Arbitrarily chosen. {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. - {Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false}, + {Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false}, // Arbitrarily chosen. {Name: "active", Type: migrator.DB_Bool, Nullable: false}, - {Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false}, + {Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false}, // Arbitrarily chosen. {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, {Name: "created", Type: migrator.DB_DateTime, Nullable: false}, {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, }, - Indices: []*migrator.Index{}, // TODO: add indexes based on the queries we make. - }) + Indices: []*migrator.Index{}, + } + tables = append(tables, dataKeyTable) - tables = append(tables, migrator.Table{ + encryptedValueTable := migrator.Table{ Name: TableNameEncryptedValue, Columns: []*migrator.Column{ {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. - {Name: "uid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, + {Name: "version", Type: migrator.DB_BigInt, Nullable: false}, {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, }, - Indices: []*migrator.Index{}, // TODO: add indexes based on the queries we make. - }) + Indices: []*migrator.Index{ + {Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}, + }, + } + tables = append(tables, encryptedValueTable) // Initialize all tables for t := range tables { @@ -138,4 +142,82 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) { mg.AddMigration(fmt.Sprintf("create table %s, index: %d", tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i])) } } + + mg.AddMigration("create index for list on "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"namespace", "active", "updated"}, + Type: migrator.IndexType, + })) + + mg.AddMigration("create index for list and read current on "+TableNameDataKey, migrator.NewAddIndexMigration(dataKeyTable, &migrator.Index{ + Cols: []string{"namespace", "label", "active"}, + Type: migrator.IndexType, + })) + + // Owner Reference columns + mg.AddMigration("add owner_reference_api_group column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_api_group", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_api_version column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_api_version", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_kind column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_kind", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_name column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_name", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "lease_token", + Type: migrator.DB_NVarchar, + Length: 36, + Nullable: true, + })) + mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"lease_token"}, + })) + mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "lease_created", + Type: migrator.DB_BigInt, + Nullable: false, + Default: "0", + })) + mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"lease_created"}, + })) + + encryptedValueTableUniqueKey := migrator.Index{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex} + updatedEncryptedValueTable := migrator.Table{ + Name: TableNameEncryptedValue, + Columns: []*migrator.Column{ + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, // Limit enforced by K8s. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, + {Name: "version", Type: migrator.DB_BigInt, Nullable: false, IsPrimaryKey: true}, + {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, + {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, + // {Name: "data_key_id", Type: migrator.DB_NVarchar, Length: 100, Nullable: false, Default: "''"}, // TODO: Not present until Grafana 12.3. + }, + PrimaryKeys: []string{"namespace", "name", "version"}, + // TODO: Not present until Grafana 12.3 + //Indices: []*migrator.Index{ + // {Cols: []string{"data_key_id"}}, + //}, + } + migrator.ConvertUniqueKeyToPrimaryKey(mg, encryptedValueTableUniqueKey, updatedEncryptedValueTable) } diff --git a/pkg/storage/unified/sql/db/migrations/resource_mig.go b/pkg/storage/unified/sql/db/migrations/resource_mig.go index 6bd32b5a0d8..0402177ef1a 100644 --- a/pkg/storage/unified/sql/db/migrations/resource_mig.go +++ b/pkg/storage/unified/sql/db/migrations/resource_mig.go @@ -167,5 +167,17 @@ func initResourceTables(mg *migrator.Migrator) string { Name: "IDX_resource_history_namespace_group_resource_name_generation", })) + oldResourceVersionUniqueKey := migrator.Index{Cols: []string{"group", "resource"}, Type: migrator.UniqueIndex} + updatedResourceVersionTable := migrator.Table{ + Name: "resource_version", + Columns: []*migrator.Column{ + {Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true}, + {Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true}, + {Name: "resource_version", Type: migrator.DB_BigInt, Nullable: false}, + }, + PrimaryKeys: []string{"group", "resource"}, + } + + migrator.ConvertUniqueKeyToPrimaryKey(mg, oldResourceVersionUniqueKey, updatedResourceVersionTable) return marker }