Convert unique keys in file and file_meta tables into primary key. (#112269)

* Convert unique keys in file and file_meta tables into primary key.

* Fix panic.

* Fix comment.

* Always add migration to drop auto-generated PK.

* Drop and create PK for mysql in single statement.

* Drop my_row_id column too. (Please drop primary key column to be able to drop generated invisible primary key.)

pkg/infra/filestorage/fs_integration_test.go

@@ -176,9 +176,6 @@ func runTests(createCases func() []fsTestCase, t *testing.T) {
 }
 
 func TestIntegrationFsStorage(t *testing.T) {
-	if true {
-		t.Skip("flakey tests - skipping")
-	}
 	testutil.SkipIntegrationTestInShortMode(t)
 
 	//skipTest := true

pkg/services/sqlstore/migrations/db_file_storage.go

@@ -1,6 +1,8 @@
 package migrations
 
-import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+import (
+	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+)
 
 func addDbFileStorageMigration(mg *migrator.Migrator) {
 	filesTable := migrator.Table{
@@ -67,4 +69,133 @@ func addDbFileStorageMigration(mg *migrator.Migrator) {
 
 	mg.AddMigration("migrate contents column to mediumblob for MySQL", migrator.NewRawSQLMigration("").
 		Mysql("ALTER TABLE file MODIFY contents MEDIUMBLOB;"))
+
+	convertFilePathHashIndexToPrimaryKey(mg)
+	convertFileMetaPathHashKeyIndexToPrimaryKey(mg)
+}
+
+func deleteAutogeneratedIds(mg *migrator.Migrator) bool {
+	// Check if delete_auto_gen_ids is enabled in the configuration
+	if mg.Cfg == nil || mg.Cfg.Raw == nil {
+		return false
+	}
+
+	deleteAutoGenIDs := mg.Cfg.Raw.Section("database").Key("delete_auto_gen_ids").MustBool(false)
+	return deleteAutoGenIDs
+}
+
+// This converts the existing unique constraint UQE_file_path_hash to a primary key in file table
+func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) {
+	// Run the migration to drop the auto-generated "my_row_id" primary key
+	// Auto-generated primary keys are a MySQL feature, so we don't need to do anything for Postgres or SQLite
+	mysql := `
+		ALTER TABLE file
+		DROP INDEX UQE_file_path_hash,
+		ADD PRIMARY KEY (path_hash);
+	`
+	if deleteAutogeneratedIds(mg) {
+		mysql = `
+			ALTER TABLE file
+			DROP PRIMARY KEY,
+			DROP COLUMN my_row_id,
+			DROP INDEX UQE_file_path_hash,
+			ADD PRIMARY KEY (path_hash);
+		`
+	}
+
+	migration := migrator.NewRawSQLMigration("").
+		Mysql(mysql).
+		Postgres(`
+			DO $$
+			BEGIN
+				-- Drop the unique constraint if it exists
+				DROP INDEX IF EXISTS "UQE_file_path_hash";
+
+				-- Add primary key if it doesn't already exist
+				IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN
+					ALTER TABLE file ADD PRIMARY KEY (path_hash);
+				END IF;
+			END $$;
+		`).SQLite(`
+			-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file" command after running migration.
+			CREATE TABLE file_new
+			(
+				path                    TEXT     NOT NULL,
+				path_hash               TEXT     NOT NULL,
+				parent_folder_path_hash TEXT     NOT NULL,
+				contents                BLOB     NOT NULL,
+				etag                    TEXT     NOT NULL,
+				cache_control           TEXT     NOT NULL,
+				content_disposition     TEXT     NOT NULL,
+				updated                 DATETIME NOT NULL,
+				created                 DATETIME NOT NULL,
+				size                    INTEGER  NOT NULL,
+				mime_type               TEXT     NOT NULL,
+
+				PRIMARY KEY (path_hash)
+			);
+
+			INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type)
+			SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file;
+
+			DROP TABLE file;
+			ALTER TABLE file_new RENAME TO file;
+
+			CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash);
+		`)
+
+	mg.AddMigration("add primary key to file table", migration)
+}
+
+// This converts the existing unique constraint UQE_file_meta_path_hash_key to a primary key in file_meta table
+func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) {
+	// Run the migration to drop the auto-generated "my_row_id" primary key
+	// Auto-generated primary keys are a MySQL feature, so we don't need to do anything for Postgres or SQLite
+	mysql := `
+		ALTER TABLE file_meta
+		DROP INDEX UQE_file_meta_path_hash_key,
+		ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
+	`
+	if deleteAutogeneratedIds(mg) {
+		mysql = `
+			ALTER TABLE file_meta
+			DROP PRIMARY KEY,
+			DROP COLUMN my_row_id,
+			DROP INDEX UQE_file_meta_path_hash_key,
+			ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
+		`
+	}
+
+	migration := migrator.NewRawSQLMigration("").
+		Mysql(mysql).
+		Postgres(`
+			DO $$
+			BEGIN
+				-- Drop the unique constraint if it exists
+				DROP INDEX IF EXISTS "UQE_file_meta_path_hash_key";
+
+				-- Add primary key if it doesn't already exist
+				IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file_meta'::regclass AND indisprimary) THEN
+					ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
+				END IF;
+			END $$;
+		`).SQLite(`
+			-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file_meta" command after running migration.
+			CREATE TABLE file_meta_new
+			(
+				path_hash TEXT NOT NULL,
+				key       TEXT NOT NULL,
+				value     TEXT NOT NULL,
+
+				PRIMARY KEY (path_hash, key)
+			);
+
+			INSERT INTO file_meta_new (path_hash, key, value)
+			SELECT path_hash, key, value FROM file_meta;
+
+			DROP TABLE file_meta;
+			ALTER TABLE file_meta_new RENAME TO file_meta;
+		`)
+
+	mg.AddMigration("add primary key to file_meta table", migration)
 }