Progress on database schema migration for account -> org refactor

This commit is contained in:
Torkel Ödegaard
2015-02-24 11:46:34 +01:00
parent da41d99aa7
commit 02a89c752b
16 changed files with 595 additions and 193 deletions
@@ -0,0 +1,72 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addApiKeyMigrations(mg *Migrator) {
mg.AddMigration("create api_key table", new(AddTableMigration).
Name("api_key").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "account_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "key", Type: DB_Varchar, Length: 64, Nullable: false},
&Column{Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- indexes ------------------
mg.AddMigration("add index api_key.account_id", new(AddIndexMigration).
Table("api_key").Columns("account_id"))
mg.AddMigration("add index api_key.key", new(AddIndexMigration).
Table("api_key").Columns("key").Unique())
mg.AddMigration("add index api_key.account_id_name", new(AddIndexMigration).
Table("api_key").Columns("account_id", "name").Unique())
// ---------------------
// account -> org changes
//------- drop indexes ------------------
mg.AddMigration("drop index api_key.account_id", new(DropIndexMigration).
Table("api_key").Columns("account_id"))
mg.AddMigration("drop index api_key.key", new(DropIndexMigration).
Table("api_key").Columns("key").Unique())
mg.AddMigration("drop index api_key.account_id_name", new(DropIndexMigration).
Table("api_key").Columns("account_id", "name").Unique())
//------- rename table ------------------
mg.AddMigration("rename table api_key to api_key_old", new(RenameTableMigration).
Rename("api_key", "api_key_old"))
//------- recreate table with new column names ------------------
mg.AddMigration("create api_key table v2", new(AddTableMigration).
Name("api_key").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "org_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "key", Type: DB_Varchar, Length: 64, Nullable: false},
&Column{Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- recreate indexes ------------------
mg.AddMigration("add index api_key.org_id", new(AddIndexMigration).
Table("api_key").Columns("org_id"))
mg.AddMigration("add index api_key.key v2", new(AddIndexMigration).
Table("api_key").Columns("key").Unique())
mg.AddMigration("add index api_key.org_id_name", new(AddIndexMigration).
Table("api_key").Columns("org_id", "name").Unique())
//------- copy data from old api_key_old -------------------
mg.AddMigration("copy data from old api_key table", new(CopyTableDataMigration).
Source("api_key_old", "id, account_id, name, key, role, created, updated").
Target("api_key", "id, org_id, name, key, role, created, updated"))
mg.AddMigration("Drop old table api_key_old", new(DropTableMigration).Table("api_key_old"))
}
@@ -0,0 +1,81 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addDashboardMigration(mg *Migrator) {
mg.AddMigration("create dashboard table", new(AddTableMigration).
Name("dashboard").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "data", Type: DB_Text, Nullable: false},
&Column{Name: "account_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
mg.AddMigration("create dashboard_tag table", new(AddTableMigration).
Name("dashboard_tag").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
))
//------- indexes ------------------
mg.AddMigration("add index dashboard.account_id", new(AddIndexMigration).
Table("dashboard").Columns("account_id"))
mg.AddMigration("add unique index dashboard_account_id_slug", new(AddIndexMigration).
Table("dashboard").Columns("account_id", "slug").Unique())
mg.AddMigration("add unique index dashboard_tag.dasboard_id_term", new(AddIndexMigration).
Table("dashboard_tag").Columns("dashboard_id", "term").Unique())
// ---------------------
// account -> org changes
//------- drop indexes ------------------
mg.AddMigration("drop index dashboard.account_id", new(DropIndexMigration).
Table("dashboard").Columns("account_id"))
mg.AddMigration("drop unique index dashboard_account_id_slug", new(DropIndexMigration).
Table("dashboard").Columns("account_id", "slug").Unique())
mg.AddMigration("drop unique index dashboard_tag.dasboard_id_term", new(DropIndexMigration).
Table("dashboard_tag").Columns("dashboard_id", "term").Unique())
//------- rename table ------------------
mg.AddMigration("rename table dashboard to dashboard_old", new(RenameTableMigration).
Rename("dashboard", "dashboard_old"))
//------- recreate table with new column names ------------------
mg.AddMigration("create dashboard table v2", new(AddTableMigration).
Name("dashboard").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "data", Type: DB_Text, Nullable: false},
&Column{Name: "org_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- dashboard table indexes ------------------
mg.AddMigration("add index dashboard.org_id", new(AddIndexMigration).
Table("dashboard").Columns("org_id"))
mg.AddMigration("add unique index dashboard_org_id_slug", new(AddIndexMigration).
Table("dashboard").Columns("org_id", "slug").Unique())
mg.AddMigration("add unique index dashboard_tag.dasboard_id_term v2", new(AddIndexMigration).
Table("dashboard_tag").Columns("dashboard_id", "term").Unique())
//------- copy data from table -------------------
mg.AddMigration("copy data from dashboard_old table", new(CopyTableDataMigration).
Source("dashboard_old", "id, version, slug, title, data, account_id, created, updated").
Target("dashboard", "id, version, slug, title, data, org_id, created, updated"))
mg.AddMigration("Drop old table dashboard_old", new(DropTableMigration).Table("dashboard_old"))
}
@@ -0,0 +1,82 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addDataSourceMigration(mg *Migrator) {
mg.AddMigration("create data_source table", new(AddTableMigration).
Name("data_source").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "account_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "user", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "database", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "basic_auth", Type: DB_Bool, Nullable: false},
&Column{Name: "basic_auth_user", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "basic_auth_password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "is_default", Type: DB_Bool, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- indexes ------------------
mg.AddMigration("add index data_source.account_id", new(AddIndexMigration).
Table("data_source").Columns("account_id"))
mg.AddMigration("add unique index data_source.account_id_name", new(AddIndexMigration).
Table("data_source").Columns("account_id", "name").Unique())
// ---------------------
// account -> org changes
//------- drop indexes ------------------
mg.AddMigration("drop index data_source.account_id", new(DropIndexMigration).
Table("data_source").Columns("account_id"))
mg.AddMigration("drop unique index data_source.account_id_name", new(DropIndexMigration).
Table("data_source").Columns("account_id", "name").Unique())
//------- rename table ------------------
mg.AddMigration("rename table data_source to data_source_old", new(RenameTableMigration).
Rename("data_source", "data_source_old"))
//------- recreate table with new column names ------------------
mg.AddMigration("create data_source table v2", new(AddTableMigration).
Name("data_source").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "org_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "user", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "database", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "basic_auth", Type: DB_Bool, Nullable: false},
&Column{Name: "basic_auth_user", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "basic_auth_password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "is_default", Type: DB_Bool, Nullable: false},
&Column{Name: "json_data", Type: DB_Text, Nullable: true},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- data_source table indexes ------------------
mg.AddMigration("add index data_source.org_id", new(AddIndexMigration).
Table("data_source").Columns("org_id"))
mg.AddMigration("add unique index data_source.org_id_name", new(AddIndexMigration).
Table("data_source").Columns("org_id", "name").Unique())
//------- copy data from table -------------------
mg.AddMigration("copy data from data_source_old table", new(CopyTableDataMigration).
Source("data_source_old", "id, account_id, version, type, name, access, url, password, user, database, basic_auth, basic_auth_user, basic_auth_password, is_default, created, updated").
Target("data_source", "id, org_id, version, type, name, access, url, password, user, database, basic_auth, basic_auth_user, basic_auth_password, is_default, created, updated"))
mg.AddMigration("Drop old table data_source_old", new(DropTableMigration).Table("data_source_old"))
}
@@ -0,0 +1,42 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
// --- Migration Guide line ---
// 1. Never change a migration that is committed and pushed to master
// 2. Always add new migrations (to change or undo previous migrations)
// 3. Some migraitons are not yet written (rename column, table, drop table, index etc)
func AddMigrations(mg *Migrator) {
addMigrationLogMigrations(mg)
addUserMigrations(mg)
addStarMigrations(mg)
addOrgMigrations(mg)
addDashboardMigration(mg)
addDataSourceMigration(mg)
addApiKeyMigrations(mg)
}
func addMigrationLogMigrations(mg *Migrator) {
mg.AddMigration("create migration_log table", new(AddTableMigration).
Name("migration_log").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "migration_id", Type: DB_NVarchar, Length: 255},
&Column{Name: "sql", Type: DB_Text},
&Column{Name: "success", Type: DB_Bool},
&Column{Name: "error", Type: DB_Text},
&Column{Name: "timestamp", Type: DB_DateTime},
))
}
func addStarMigrations(mg *Migrator) {
mg.AddMigration("create star table", new(AddTableMigration).
Name("star").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "user_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
))
mg.AddMigration("add unique index star.user_id_dashboard_id", new(AddIndexMigration).
Table("star").Columns("user_id", "dashboard_id").Unique())
}
@@ -0,0 +1,60 @@
package migrations
import (
"fmt"
"strings"
"testing"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/log"
. "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
. "github.com/smartystreets/goconvey/convey"
)
var indexTypes = []string{"Unknown", "INDEX", "UNIQUE INDEX"}
func TestMigrations(t *testing.T) {
log.NewLogger(0, "console", `{"level": 0}`)
testDBs := []sqlutil.TestDB{
sqlutil.TestDB_Sqlite3,
}
for _, testDB := range testDBs {
Convey("Initial "+testDB.DriverName+" migration", t, func() {
x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr)
So(err, ShouldBeNil)
sqlutil.CleanDB(x)
mg := NewMigrator(x)
mg.LogLevel = log.DEBUG
addMigrations(mg)
err = mg.Start()
So(err, ShouldBeNil)
tables, err := x.DBMetas()
So(err, ShouldBeNil)
fmt.Printf("\nDB Schema after migration: table count: %v\n", len(tables))
for _, table := range tables {
fmt.Printf("\nTable: %v \n", table.Name)
for _, column := range table.Columns() {
fmt.Printf("\t %v \n", column.String(x.Dialect()))
}
if len(table.Indexes) > 0 {
fmt.Printf("\n\tIndexes:\n")
for _, index := range table.Indexes {
fmt.Printf("\t %v (%v) %v \n", index.Name, strings.Join(index.Cols, ","), indexTypes[index.Type])
}
}
}
})
}
}
@@ -0,0 +1,55 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addOrgMigrations(mg *Migrator) {
//------- org table -------------------
mg.AddMigration("create org table", new(AddTableMigration).
Name("org").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "address2", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "city", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "state", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "zip_code", Type: DB_NVarchar, Length: 50, Nullable: true},
&Column{Name: "country", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "billing_email", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- indices -------------------
mg.AddMigration("add unique index org.name", new(AddIndexMigration).
Table("org").Columns("name").Unique())
//------- org_user table -------------------
mg.AddMigration("create org_user table", new(AddTableMigration).
Name("org_user").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "org_id", Type: DB_BigInt},
&Column{Name: "user_id", Type: DB_BigInt},
&Column{Name: "role", Type: DB_NVarchar, Length: 20},
&Column{Name: "created", Type: DB_DateTime},
&Column{Name: "updated", Type: DB_DateTime},
))
//------- indices -------------------
mg.AddMigration("add unique index org_user_aid_uid", new(AddIndexMigration).
Name("org_user_aid_uid").Table("org_user").Columns("org_id", "user_id").Unique())
//------- TEMP table rename / copy data -------------------
mg.AddMigration("copy data from old account table", new(CopyTableDataMigration).
Source("account", "id, version, name, created, updated").
Target("org", "id, version, name, created, updated").
IfTableExists("account"))
mg.AddMigration("copy data from old account_user table", new(CopyTableDataMigration).
Source("account_user", "id, account_id, user_id, role, created, updated").
Target("org_user", "id, org_id, user_id, role, created, updated").
IfTableExists("account_user"))
mg.AddMigration("Drop old table account", new(DropTableMigration).Table("account"))
mg.AddMigration("Drop old table account_user", new(DropTableMigration).Table("account_user"))
}
@@ -0,0 +1,84 @@
package migrations
import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addUserMigrations(mg *Migrator) {
mg.AddMigration("create user table", new(AddTableMigration).
Name("user").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true},
&Column{Name: "rands", Type: DB_NVarchar, Length: 50, Nullable: true},
&Column{Name: "company", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "account_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "is_admin", Type: DB_Bool, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
mg.AddMigration("Add email_verified flag", new(AddColumnMigration).
Table("user").Column(&Column{Name: "email_verified", Type: DB_Bool, Nullable: true}))
mg.AddMigration("Add user.theme column", new(AddColumnMigration).
Table("user").Column(&Column{Name: "theme", Type: DB_Varchar, Nullable: true, Length: 20}))
//------- user table indexes ------------------
mg.AddMigration("add unique index user.login", new(AddIndexMigration).
Table("user").Columns("login").Unique())
mg.AddMigration("add unique index user.email", new(AddIndexMigration).
Table("user").Columns("email").Unique())
// ---------------------
// account -> org changes
//------- drop indexes ------------------
mg.AddMigration("drop unique index user.login", new(DropIndexMigration).
Table("user").Columns("login").Unique())
mg.AddMigration("drop unique index user.email", new(DropIndexMigration).
Table("user").Columns("email").Unique())
//------- rename table ------------------
mg.AddMigration("rename table user to user_old", new(RenameTableMigration).
Rename("user", "user_old"))
//------- recreate table with new column names ------------------
mg.AddMigration("create user table v2", new(AddTableMigration).
Name("user").WithColumns(
&Column{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
&Column{Name: "version", Type: DB_Int, Nullable: false},
&Column{Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false},
&Column{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true},
&Column{Name: "rands", Type: DB_NVarchar, Length: 50, Nullable: true},
&Column{Name: "company", Type: DB_NVarchar, Length: 255, Nullable: true},
&Column{Name: "org_id", Type: DB_BigInt, Nullable: false},
&Column{Name: "email_verified", Type: DB_Bool, Nullable: true},
&Column{Name: "theme", Type: DB_NVarchar, Nullable: true},
&Column{Name: "is_admin", Type: DB_Bool, Nullable: false},
&Column{Name: "created", Type: DB_DateTime, Nullable: false},
&Column{Name: "updated", Type: DB_DateTime, Nullable: false},
))
//------- user table indexes ------------------
mg.AddMigration("add unique index user.login v2", new(AddIndexMigration).
Table("user").Columns("login").Unique())
mg.AddMigration("add unique index user.email v2", new(AddIndexMigration).
Table("user").Columns("email").Unique())
//------- copy data from user_old table -------------------
mg.AddMigration("copy data from user_old table", new(CopyTableDataMigration).
Source("user_old", "id, version, login, email, name, password, salt, rands, company, account_id, is_admin, created, updated").
Target("user", "id, version, login, email, name, password, salt, rands, company, org_id, is_admin, created, updated"))
mg.AddMigration("Drop old table user_old", new(DropTableMigration).Table("user_old"))
}