From 40fc660cc8e1c9e25468c410809817620f9b3f19 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 12:41:18 -0500 Subject: [PATCH] [release-11.6.8] fix: file and file_meta migrations (#113280) fix: file and file_meta migrations (#112611) * fix file and file_meta migrations to check the database state to decide which migration to run (cherry picked from commit 545b7bf8ff49ae1427cf24eeb38f599612299e8e) Co-authored-by: Will Assis <35489495+gassiss@users.noreply.github.com> --- .../sqlstore/migrations/db_file_storage.go | 135 ++++++++++++++++++ pkg/services/sqlstore/migrator/conditions.go | 25 ++++ 2 files changed, 160 insertions(+) diff --git a/pkg/services/sqlstore/migrations/db_file_storage.go b/pkg/services/sqlstore/migrations/db_file_storage.go index fcc49ae0d57..e8ca9348455 100644 --- a/pkg/services/sqlstore/migrations/db_file_storage.go +++ b/pkg/services/sqlstore/migrations/db_file_storage.go @@ -67,4 +67,139 @@ func addDbFileStorageMigration(mg *migrator.Migrator) { mg.AddMigration("migrate contents column to mediumblob for MySQL", migrator.NewRawSQLMigration(""). Mysql("ALTER TABLE file MODIFY contents MEDIUMBLOB;")) + + convertFilePathHashIndexToPrimaryKey(mg) + convertFileMetaPathHashKeyIndexToPrimaryKey(mg) +} + +// This converts the existing unique constraint UQE_file_path_hash to a primary key in file table +func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) { + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do everything in one sql statement + mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(` + ALTER TABLE file + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_path_hash, + ADD PRIMARY KEY (path_hash); + `) + mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file", ColumnName: "my_row_id"} + mg.AddMigration("drop my_row_id and add primary key to file table if my_row_id exists (auto-generated mysql column)", mysqlMigration1) + + mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file DROP INDEX UQE_file_path_hash`) + mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file", IndexName: "UQE_file_path_hash"} + mg.AddMigration("drop file_path unique index from file table if it exists (mysql)", mysqlMigration2) + + mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file ADD PRIMARY KEY (path_hash);`) + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file", ColumnName: "path_hash"} + mg.AddMigration("add primary key to file table if it doesn't exist (mysql)", mysqlMigration3) + + postgres := ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_path_hash"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN + ALTER TABLE file ADD PRIMARY KEY (path_hash); + END IF; + END $$; + ` + + sqlite := ` + -- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file" command after running migration. + CREATE TABLE file_new + ( + path TEXT NOT NULL, + path_hash TEXT NOT NULL, + parent_folder_path_hash TEXT NOT NULL, + contents BLOB NOT NULL, + etag TEXT NOT NULL, + cache_control TEXT NOT NULL, + content_disposition TEXT NOT NULL, + updated DATETIME NOT NULL, + created DATETIME NOT NULL, + size INTEGER NOT NULL, + mime_type TEXT NOT NULL, + + PRIMARY KEY (path_hash) + ); + + INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type) + SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file; + + DROP TABLE file; + ALTER TABLE file_new RENAME TO file; + + CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash); + ` + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + migration := migrator.NewRawSQLMigration(""). + Postgres(postgres). + SQLite(sqlite) + + mg.AddMigration("add primary key to file table (postgres and sqlite)", migration) +} + +// This converts the existing unique constraint UQE_file_meta_path_hash_key to a primary key in file_meta table +func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) { + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do everything in one sql statement + mysqlMigration1 := migrator.NewRawSQLMigration("").Mysql(` + ALTER TABLE file_meta + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_meta_path_hash_key, + ADD PRIMARY KEY (path_hash, ` + "`key`" + `); + `) + mysqlMigration1.Condition = &migrator.IfColumnExistsCondition{TableName: "file_meta", ColumnName: "my_row_id"} + mg.AddMigration("drop my_row_id and add primary key to file_meta table if my_row_id exists (auto-generated mysql column)", mysqlMigration1) + + mysqlMigration2 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta DROP INDEX UQE_file_meta_path_hash_key`) + mysqlMigration2.Condition = &migrator.IfIndexExistsCondition{TableName: "file_meta", IndexName: "UQE_file_meta_path_hash_key"} + mg.AddMigration("drop file_path unique index from file_meta table if it exists (mysql)", mysqlMigration2) + + mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);`) + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta", ColumnName: "path_hash"} + mg.AddMigration("add primary key to file_meta table if it doesn't exist (mysql)", mysqlMigration3) + + postgres := ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_meta_path_hash_key"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file_meta'::regclass AND indisprimary) THEN + ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `); + END IF; + END $$; + ` + + sqlite := ` + -- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file_meta" command after running migration. + CREATE TABLE file_meta_new + ( + path_hash TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + + PRIMARY KEY (path_hash, key) + ); + + INSERT INTO file_meta_new (path_hash, key, value) + SELECT path_hash, key, value FROM file_meta; + + DROP TABLE file_meta; + ALTER TABLE file_meta_new RENAME TO file_meta; + ` + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + migration := migrator.NewRawSQLMigration(""). + Postgres(postgres). + SQLite(sqlite) + + mg.AddMigration("add primary key to file_meta table (postgres and sqlite)", migration) } diff --git a/pkg/services/sqlstore/migrator/conditions.go b/pkg/services/sqlstore/migrator/conditions.go index 4fc7c780e84..b751be61c0e 100644 --- a/pkg/services/sqlstore/migrator/conditions.go +++ b/pkg/services/sqlstore/migrator/conditions.go @@ -46,3 +46,28 @@ type IfColumnNotExistsCondition struct { func (c *IfColumnNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) { return dialect.ColumnCheckSQL(c.TableName, c.ColumnName) } + +type IfColumnExistsCondition struct { + ExistsMigrationCondition + TableName string + ColumnName string +} + +func (c *IfColumnExistsCondition) SQL(dialect Dialect) (string, []interface{}) { + return dialect.ColumnCheckSQL(c.TableName, c.ColumnName) +} + +type IfPrimaryKeyNotExistsCondition struct { + NotExistsMigrationCondition + TableName string + ColumnName string +} + +func (c *IfPrimaryKeyNotExistsCondition) SQL(dialect Dialect) (string, []interface{}) { + // only use it with mysql + if dialect.DriverName() == "mysql" { + return "SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME=? AND COLUMN_KEY='PRI'", []interface{}{c.TableName} + } + + return "", nil +}