diff --git a/pkg/services/sqlstore/migrations/builder.go b/pkg/services/sqlstore/migrations/builder.go index 0b1accae8dd..06fc7711b80 100644 --- a/pkg/services/sqlstore/migrations/builder.go +++ b/pkg/services/sqlstore/migrations/builder.go @@ -13,6 +13,8 @@ const ( type Migration interface { Sql(dialect Dialect) string + Id() string + SetId(string) } type ColumnType string @@ -22,7 +24,15 @@ const ( ) type MigrationBase struct { - desc string + id string +} + +func (m *MigrationBase) Id() string { + return m.id +} + +func (m *MigrationBase) SetId(id string) { + m.id = id } type RawSqlMigration struct { @@ -53,11 +63,6 @@ func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration { return m } -func (m *RawSqlMigration) Desc(desc string) *RawSqlMigration { - m.desc = desc - return m -} - type AddColumnMigration struct { MigrationBase tableName string @@ -90,16 +95,12 @@ func (m *AddColumnMigration) Sql(dialect Dialect) string { return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", m.tableName, m.columnName, dialect.ToDBTypeSql(m.columnType, m.length)) } -func (m *AddColumnMigration) Desc(desc string) *AddColumnMigration { - m.desc = desc - return m -} - type AddIndexMigration struct { MigrationBase tableName string columns string indexName string + unique string } func (m *AddIndexMigration) Name(name string) *AddIndexMigration { @@ -112,11 +113,16 @@ func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration { return m } +func (m *AddIndexMigration) Unique() *AddIndexMigration { + m.unique = "UNIQUE" + return m +} + func (m *AddIndexMigration) Columns(columns ...string) *AddIndexMigration { m.columns = strings.Join(columns, ",") return m } func (m *AddIndexMigration) Sql(dialect Dialect) string { - return fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s(%s)", m.indexName, m.tableName, m.columns) + return fmt.Sprintf("CREATE %s INDEX %s ON %s(%s)", m.unique, m.indexName, m.tableName, m.columns) } diff --git a/pkg/services/sqlstore/migrations/engine.go b/pkg/services/sqlstore/migrations/engine.go deleted file mode 100644 index b6362ff2abf..00000000000 --- a/pkg/services/sqlstore/migrations/engine.go +++ /dev/null @@ -1,99 +0,0 @@ -package migrations - -import ( - _ "github.com/go-sql-driver/mysql" - "github.com/go-xorm/xorm" - _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" - "github.com/torkelo/grafana-pro/pkg/log" -) - -var x *xorm.Engine -var dialect Dialect - -func getSchemaVersion() (int, error) { - exists, err := x.IsTableExist(new(SchemaVersion)) - if err != nil { - return 0, err - } - - if !exists { - if err := x.CreateTables(new(SchemaVersion)); err != nil { - return 0, err - } - return 0, nil - } - - v := SchemaVersion{} - _, err = x.Table("schema_version").Limit(1, 0).Desc("version").Get(&v) - return v.Version, err -} - -func setEngineAndDialect(engine *xorm.Engine) { - x = engine - switch x.DriverName() { - case MYSQL: - dialect = new(Mysql) - case SQLITE: - dialect = new(Sqlite3) - } -} - -func StartMigration(engine *xorm.Engine) error { - log.Info("Starting database schema migration: DB: %v", engine.DriverName()) - - setEngineAndDialect(engine) - - _, err := getSchemaVersion() - if err != nil { - return err - } - - for _, m := range migrationList { - if err := execMigration(m); err != nil { - return err - } - } - - return nil -} - -func execMigration(m Migration) error { - err := inTransaction(func(sess *xorm.Session) error { - _, err := sess.Exec(m.Sql(dialect)) - if err != nil { - return err - } - return nil - }) - - if err != nil { - return err - } - - return nil -} - -type dbTransactionFunc func(sess *xorm.Session) error - -func inTransaction(callback dbTransactionFunc) error { - var err error - - sess := x.NewSession() - defer sess.Close() - - if err = sess.Begin(); err != nil { - return err - } - - err = callback(sess) - - if err != nil { - sess.Rollback() - return err - } else if err = sess.Commit(); err != nil { - return err - } - - return nil -} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 0e261ecb01c..32cc7f8e8e6 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -1,6 +1,6 @@ package migrations -var migrationList []Migration +import "time" // Id int64 // Login string `xorm:"UNIQUE NOT NULL"` @@ -16,9 +16,11 @@ var migrationList []Migration // Created time.Time // Updated time.Time -func init() { - // ------------------------------ - addMigration(new(RawSqlMigration).Desc("Create account table"). +func AddMigrations(mg *Migrator) { + + // TABLE Account + // ------------------------------- + mg.AddMigration("create account table", new(RawSqlMigration). Sqlite(` CREATE TABLE account ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -34,25 +36,18 @@ func init() { ) `)) // ------------------------------ - addMigration(new(AddIndexMigration). + mg.AddMigration("add index UIX_account.login", new(AddIndexMigration). Name("UIX_account_login").Table("account").Columns("login")) // ------------------------------ - addMigration(new(AddColumnMigration).Desc("Add name column"). + mg.AddMigration("add column", new(AddColumnMigration). Table("account").Column("name").Type(DB_TYPE_STRING).Length(255)) } -func addMigration(m Migration) { - migrationList = append(migrationList, m) -} - -type SchemaVersion struct { - Version int -} - -type SchemaLog struct { - Id int64 - Version int64 - Desc string - Info string - Error bool +type MigrationLog struct { + Id int64 + MigrationId string + Sql string + Success bool + Error string + Timestamp time.Time } diff --git a/pkg/services/sqlstore/migrations/migrations_test.go b/pkg/services/sqlstore/migrations/migrations_test.go index 247ebd6ca68..e485292f4a4 100644 --- a/pkg/services/sqlstore/migrations/migrations_test.go +++ b/pkg/services/sqlstore/migrations/migrations_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/go-xorm/xorm" + "github.com/torkelo/grafana-pro/pkg/log" . "github.com/smartystreets/goconvey/convey" ) @@ -31,8 +32,10 @@ func cleanDB(x *xorm.Engine) { var indexTypes = []string{"Unknown", "", "UNIQUE"} func TestMigrations(t *testing.T) { + log.NewLogger(0, "console", `{"level": 0}`) + testDBs := [][]string{ - //[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"}, + []string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"}, []string{"sqlite3", ":memory:"}, } @@ -46,7 +49,10 @@ func TestMigrations(t *testing.T) { cleanDB(x) } - err = StartMigration(x) + mg := NewMigrator(x) + AddMigrations(mg) + + err = mg.Start() So(err, ShouldBeNil) tables, err := x.DBMetas() diff --git a/pkg/services/sqlstore/migrations/migrator.go b/pkg/services/sqlstore/migrations/migrator.go new file mode 100644 index 00000000000..8bfc003e7a4 --- /dev/null +++ b/pkg/services/sqlstore/migrations/migrator.go @@ -0,0 +1,143 @@ +package migrations + +import ( + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/go-xorm/xorm" + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" + "github.com/torkelo/grafana-pro/pkg/log" +) + +type Migrator struct { + x *xorm.Engine + dialect Dialect + migrations []Migration +} + +func NewMigrator(engine *xorm.Engine) *Migrator { + mg := &Migrator{} + mg.x = engine + mg.migrations = make([]Migration, 0) + + switch mg.x.DriverName() { + case MYSQL: + mg.dialect = new(Mysql) + case SQLITE: + mg.dialect = new(Sqlite3) + } + + return mg +} + +func (mg *Migrator) AddMigration(id string, m Migration) { + m.SetId(id) + mg.migrations = append(mg.migrations, m) +} + +func (mg *Migrator) GetMigrationLog() (map[string]MigrationLog, error) { + exists, err := mg.x.IsTableExist(new(MigrationLog)) + if err != nil { + return nil, err + } + + if !exists { + if err := mg.x.CreateTables(new(MigrationLog)); err != nil { + return nil, err + } + return nil, nil + } + + logMap := make(map[string]MigrationLog) + logItems := make([]MigrationLog, 0) + if err = mg.x.Find(&logItems); err != nil { + return nil, err + } + + for _, logItem := range logItems { + if !logItem.Success { + continue + } + logMap[logItem.MigrationId] = logItem + } + + return logMap, nil +} + +func (mg *Migrator) Start() error { + log.Info("Migrator::Start DB migration") + + logMap, err := mg.GetMigrationLog() + if err != nil { + return err + } + + for _, m := range mg.migrations { + _, exists := logMap[m.Id()] + if exists { + log.Info("Migrator:: Skipping migration: %v, Already executed", m.Id()) + continue + } + + record := MigrationLog{ + MigrationId: m.Id(), + Sql: m.Sql(mg.dialect), + Timestamp: time.Now(), + } + + if err := mg.exec(m); err != nil { + record.Error = err.Error() + mg.x.Insert(&record) + return err + } else { + record.Success = true + mg.x.Insert(&record) + } + } + + return nil +} + +func (mg *Migrator) exec(m Migration) error { + log.Info("Migrator::exec migration id: %v", m.Id()) + + err := mg.inTransaction(func(sess *xorm.Session) error { + _, err := sess.Exec(m.Sql(mg.dialect)) + if err != nil { + log.Error(3, "Migrator::exec FAILED migration id: %v, err: %v", m.Id(), err) + return err + } + return nil + }) + + if err != nil { + return err + } + + return nil +} + +type dbTransactionFunc func(sess *xorm.Session) error + +func (mg *Migrator) inTransaction(callback dbTransactionFunc) error { + var err error + + sess := mg.x.NewSession() + defer sess.Close() + + if err = sess.Begin(); err != nil { + return err + } + + err = callback(sess) + + if err != nil { + sess.Rollback() + return err + } else if err = sess.Commit(); err != nil { + return err + } + + return nil +} diff --git a/pkg/services/sqlstore/migrations/migrator_test.go b/pkg/services/sqlstore/migrations/migrator_test.go new file mode 100644 index 00000000000..e5b71250176 --- /dev/null +++ b/pkg/services/sqlstore/migrations/migrator_test.go @@ -0,0 +1,78 @@ +package migrations + +import ( + "testing" + + "github.com/go-xorm/xorm" + + . "github.com/smartystreets/goconvey/convey" +) + +// func cleanDB(x *xorm.Engine) { +// tables, _ := x.DBMetas() +// sess := x.NewSession() +// defer sess.Close() +// +// for _, table := range tables { +// if _, err := sess.Exec("SET FOREIGN_KEY_CHECKS = 0"); err != nil { +// panic("Failed to disable foreign key checks") +// } +// if _, err := sess.Exec("DROP TABLE " + table.Name); err != nil { +// panic(fmt.Sprintf("Failed to delete table: %v, err: %v", table.Name, err)) +// } +// if _, err := sess.Exec("SET FOREIGN_KEY_CHECKS = 1"); err != nil { +// panic("Failed to disable foreign key checks") +// } +// } +// } +// +// var indexTypes = []string{"Unknown", "", "UNIQUE"} +// + +func TestMigrator(t *testing.T) { + + Convey("Migrator", t, func() { + x, err := xorm.NewEngine(SQLITE, ":memory:") + So(err, ShouldBeNil) + + mg := NewMigrator(x) + + Convey("Given one migration", func() { + mg.AddMigration("test migration", new(RawSqlMigration). + Sqlite(` + CREATE TABLE account ( + id INTEGER PRIMARY KEY AUTOINCREMENT + )`). + Mysql(` + CREATE TABLE account ( + id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id) + )`)) + + err := mg.Start() + So(err, ShouldBeNil) + + log, err := mg.GetMigrationLog() + So(err, ShouldBeNil) + So(len(log), ShouldEqual, 1) + }) + + // So(err, ShouldBeNil) + // + // So(len(tables), ShouldEqual, 2) + // fmt.Printf("\nDB Schema after migration: table count: %v\n", len(tables)) + // + // for _, table := range tables { + // fmt.Printf("\nTable: %v \n", table.Name) + // for _, column := range table.Columns() { + // fmt.Printf("\t %v \n", column.String(x.Dialect())) + // } + // + // if len(table.Indexes) > 0 { + // fmt.Printf("\n\tIndexes:\n") + // for _, index := range table.Indexes { + // fmt.Printf("\t %v (%v) %v \n", index.Name, strings.Join(index.Cols, ","), indexTypes[index.Type]) + // } + // } + // } + }) +}