Peter Štibraný
e290c92e1b
* Convert unique keys in file and file_meta tables into primary key. * Fix panic. * Fix comment. * Always add migration to drop auto-generated PK. * Drop and create PK for mysql in single statement. * Drop my_row_id column too. (Please drop primary key column to be able to drop generated invisible primary key.)
202 lines
7.7 KiB
Go
202 lines
7.7 KiB
Go
package migrations
|
|
|
|
import (
|
|
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
|
)
|
|
|
|
func addDbFileStorageMigration(mg *migrator.Migrator) {
|
|
filesTable := migrator.Table{
|
|
Name: "file",
|
|
Columns: []*migrator.Column{
|
|
{Name: "path", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
|
|
|
|
// path_hash is used for indexing. we are using it to circumvent the max length limit of 191 for varchar2 fields in MySQL 5.6
|
|
{Name: "path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
|
|
|
|
// parent_folder_path_hash is an optimization for a common use case - list all files in a given folder
|
|
{Name: "parent_folder_path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
|
|
|
|
{Name: "contents", Type: migrator.DB_Blob, Nullable: false},
|
|
|
|
// HTTP Entity tag; md5 hash
|
|
{Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false},
|
|
|
|
// cache_control HTTP header
|
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
|
|
{Name: "cache_control", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
|
|
|
|
// content_disposition HTTP header - inline/attachment file display
|
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
|
|
{Name: "content_disposition", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
|
|
|
|
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
|
|
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
|
|
{Name: "size", Type: migrator.DB_BigInt, Nullable: false},
|
|
{Name: "mime_type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
|
|
},
|
|
Indices: []*migrator.Index{
|
|
{Cols: []string{"path_hash"}, Type: migrator.UniqueIndex},
|
|
{Cols: []string{"parent_folder_path_hash"}},
|
|
},
|
|
}
|
|
|
|
mg.AddMigration("create file table", migrator.NewAddTableMigration(filesTable))
|
|
mg.AddMigration("file table idx: path natural pk", migrator.NewAddIndexMigration(filesTable, filesTable.Indices[0]))
|
|
mg.AddMigration("file table idx: parent_folder_path_hash fast folder retrieval", migrator.NewAddIndexMigration(filesTable, filesTable.Indices[1]))
|
|
|
|
fileMetaTable := migrator.Table{
|
|
Name: "file_meta",
|
|
Columns: []*migrator.Column{
|
|
{Name: "path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
|
|
|
|
// 191 is the maximum length of indexable VARCHAR fields in MySQL 5.6 <= with utf8mb4 encoding
|
|
{Name: "key", Type: migrator.DB_NVarchar, Length: 191, Nullable: false},
|
|
{Name: "value", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
|
|
},
|
|
Indices: []*migrator.Index{
|
|
{Cols: []string{"path_hash", "key"}, Type: migrator.UniqueIndex},
|
|
},
|
|
}
|
|
|
|
mg.AddMigration("create file_meta table", migrator.NewAddTableMigration(fileMetaTable))
|
|
mg.AddMigration("file table idx: path key", migrator.NewAddIndexMigration(fileMetaTable, fileMetaTable.Indices[0]))
|
|
|
|
// TODO: add collation support to `migrator.Column`
|
|
mg.AddMigration("set path collation in file table", migrator.NewRawSQLMigration("").
|
|
// MySQL `utf8mb4_unicode_ci` collation is set in `mysql_dialect.go`
|
|
// SQLite uses a `BINARY` collation by default
|
|
Postgres("ALTER TABLE file ALTER COLUMN path TYPE VARCHAR(1024) COLLATE \"C\";")) // Collate C - sorting done based on character code byte values
|
|
|
|
mg.AddMigration("migrate contents column to mediumblob for MySQL", migrator.NewRawSQLMigration("").
|
|
Mysql("ALTER TABLE file MODIFY contents MEDIUMBLOB;"))
|
|
|
|
convertFilePathHashIndexToPrimaryKey(mg)
|
|
convertFileMetaPathHashKeyIndexToPrimaryKey(mg)
|
|
}
|
|
|
|
func deleteAutogeneratedIds(mg *migrator.Migrator) bool {
|
|
// Check if delete_auto_gen_ids is enabled in the configuration
|
|
if mg.Cfg == nil || mg.Cfg.Raw == nil {
|
|
return false
|
|
}
|
|
|
|
deleteAutoGenIDs := mg.Cfg.Raw.Section("database").Key("delete_auto_gen_ids").MustBool(false)
|
|
return deleteAutoGenIDs
|
|
}
|
|
|
|
// This converts the existing unique constraint UQE_file_path_hash to a primary key in file table
|
|
func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) {
|
|
// Run the migration to drop the auto-generated "my_row_id" primary key
|
|
// Auto-generated primary keys are a MySQL feature, so we don't need to do anything for Postgres or SQLite
|
|
mysql := `
|
|
ALTER TABLE file
|
|
DROP INDEX UQE_file_path_hash,
|
|
ADD PRIMARY KEY (path_hash);
|
|
`
|
|
if deleteAutogeneratedIds(mg) {
|
|
mysql = `
|
|
ALTER TABLE file
|
|
DROP PRIMARY KEY,
|
|
DROP COLUMN my_row_id,
|
|
DROP INDEX UQE_file_path_hash,
|
|
ADD PRIMARY KEY (path_hash);
|
|
`
|
|
}
|
|
|
|
migration := migrator.NewRawSQLMigration("").
|
|
Mysql(mysql).
|
|
Postgres(`
|
|
DO $$
|
|
BEGIN
|
|
-- Drop the unique constraint if it exists
|
|
DROP INDEX IF EXISTS "UQE_file_path_hash";
|
|
|
|
-- Add primary key if it doesn't already exist
|
|
IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN
|
|
ALTER TABLE file ADD PRIMARY KEY (path_hash);
|
|
END IF;
|
|
END $$;
|
|
`).SQLite(`
|
|
-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file" command after running migration.
|
|
CREATE TABLE file_new
|
|
(
|
|
path TEXT NOT NULL,
|
|
path_hash TEXT NOT NULL,
|
|
parent_folder_path_hash TEXT NOT NULL,
|
|
contents BLOB NOT NULL,
|
|
etag TEXT NOT NULL,
|
|
cache_control TEXT NOT NULL,
|
|
content_disposition TEXT NOT NULL,
|
|
updated DATETIME NOT NULL,
|
|
created DATETIME NOT NULL,
|
|
size INTEGER NOT NULL,
|
|
mime_type TEXT NOT NULL,
|
|
|
|
PRIMARY KEY (path_hash)
|
|
);
|
|
|
|
INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type)
|
|
SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file;
|
|
|
|
DROP TABLE file;
|
|
ALTER TABLE file_new RENAME TO file;
|
|
|
|
CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash);
|
|
`)
|
|
|
|
mg.AddMigration("add primary key to file table", migration)
|
|
}
|
|
|
|
// This converts the existing unique constraint UQE_file_meta_path_hash_key to a primary key in file_meta table
|
|
func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) {
|
|
// Run the migration to drop the auto-generated "my_row_id" primary key
|
|
// Auto-generated primary keys are a MySQL feature, so we don't need to do anything for Postgres or SQLite
|
|
mysql := `
|
|
ALTER TABLE file_meta
|
|
DROP INDEX UQE_file_meta_path_hash_key,
|
|
ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
|
|
`
|
|
if deleteAutogeneratedIds(mg) {
|
|
mysql = `
|
|
ALTER TABLE file_meta
|
|
DROP PRIMARY KEY,
|
|
DROP COLUMN my_row_id,
|
|
DROP INDEX UQE_file_meta_path_hash_key,
|
|
ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
|
|
`
|
|
}
|
|
|
|
migration := migrator.NewRawSQLMigration("").
|
|
Mysql(mysql).
|
|
Postgres(`
|
|
DO $$
|
|
BEGIN
|
|
-- Drop the unique constraint if it exists
|
|
DROP INDEX IF EXISTS "UQE_file_meta_path_hash_key";
|
|
|
|
-- Add primary key if it doesn't already exist
|
|
IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file_meta'::regclass AND indisprimary) THEN
|
|
ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);
|
|
END IF;
|
|
END $$;
|
|
`).SQLite(`
|
|
-- For SQLite we need to recreate the table with primary key. CREATE TABLE was generated by ".schema file_meta" command after running migration.
|
|
CREATE TABLE file_meta_new
|
|
(
|
|
path_hash TEXT NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
|
|
PRIMARY KEY (path_hash, key)
|
|
);
|
|
|
|
INSERT INTO file_meta_new (path_hash, key, value)
|
|
SELECT path_hash, key, value FROM file_meta;
|
|
|
|
DROP TABLE file_meta;
|
|
ALTER TABLE file_meta_new RENAME TO file_meta;
|
|
`)
|
|
|
|
mg.AddMigration("add primary key to file_meta table", migration)
|
|
}
|