Migrations: Support independent migration log for entity api (#68511)

pkg/services/sqlstore/migrations/entity_store_mig.go

@@ -1,220 +0,0 @@
-package migrations
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
-	"github.com/grafana/grafana/pkg/setting"
-)
-
-func getLatinPathColumn(name string) *migrator.Column {
-	return &migrator.Column{
-		Name:     name,
-		Type:     migrator.DB_NVarchar,
-		Length:   1024,
-		Nullable: false,
-		IsLatin:  true, // only used in MySQL
-	}
-}
-
-func addEntityStoreMigrations(mg *migrator.Migrator) {
-	grnLength := 256 // len(tenant)~8 + len(kind)!16 + len(kind)~128 = 256
-	tables := []migrator.Table{}
-	tables = append(tables, migrator.Table{
-		Name: "entity",
-		Columns: []*migrator.Column{
-			// Object ID (OID) will be unique across all objects/instances
-			// uuid5( tenant_id, kind + uid )
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false, IsPrimaryKey: true},
-
-			// The entity identifier
-			{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			{Name: "folder", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			{Name: "slug", Type: migrator.DB_NVarchar, Length: 189, Nullable: false}, // from title
-
-			// The raw entity body (any byte array)
-			{Name: "body", Type: migrator.DB_LongBlob, Nullable: true}, // null when nested or remote
-			{Name: "size", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false, IsLatin: true}, // md5(body)
-			{Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
-
-			// Who changed what when -- We should avoid JOINs with other tables in the database
-			{Name: "updated_at", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "created_at", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "updated_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
-			{Name: "created_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
-
-			// Mark objects with origin metadata
-			{Name: "origin", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			getLatinPathColumn("origin_key"), // index with length 1024
-			{Name: "origin_ts", Type: migrator.DB_BigInt, Nullable: false},
-
-			// Summary data (always extracted from the `body` column)
-			{Name: "name", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "description", Type: migrator.DB_NVarchar, Length: 255, Nullable: true},
-			{Name: "labels", Type: migrator.DB_Text, Nullable: true}, // JSON object
-			{Name: "fields", Type: migrator.DB_Text, Nullable: true}, // JSON object
-			{Name: "errors", Type: migrator.DB_Text, Nullable: true}, // JSON object
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"kind"}},
-			{Cols: []string{"folder"}},
-			{Cols: []string{"uid"}},
-
-			{Cols: []string{"tenant_id", "kind", "uid"}, Type: migrator.UniqueIndex},
-			// {Cols: []string{"tenant_id", "folder", "slug"}, Type: migrator.UniqueIndex},
-		},
-	})
-
-	// when saving a folder, keep a path version cached (all info is derived from entity table)
-	tables = append(tables, migrator.Table{
-		Name: "entity_folder",
-		Columns: []*migrator.Column{
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false, IsPrimaryKey: true},
-			{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			getLatinPathColumn("slug_path"),                             ///slug/slug/slug/
-			{Name: "tree", Type: migrator.DB_Text, Nullable: false},     // JSON []{uid, title}
-			{Name: "depth", Type: migrator.DB_Int, Nullable: false},     // starts at 1
-			{Name: "left", Type: migrator.DB_Int, Nullable: false},      // MPTT
-			{Name: "right", Type: migrator.DB_Int, Nullable: false},     // MPTT
-			{Name: "detached", Type: migrator.DB_Bool, Nullable: false}, // a parent folder was not found
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"tenant_id", "uid"}, Type: migrator.UniqueIndex},
-			//	{Cols: []string{"tenant_id", "slug_path"}, Type: migrator.UniqueIndex},
-		},
-	})
-
-	tables = append(tables, migrator.Table{
-		Name: "entity_labels",
-		Columns: []*migrator.Column{
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
-			{Name: "label", Type: migrator.DB_NVarchar, Length: 191, Nullable: false},
-			{Name: "value", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
-			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: true},
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"grn", "label"}, Type: migrator.UniqueIndex},
-			{Cols: []string{"parent_grn"}, Type: migrator.IndexType},
-		},
-	})
-
-	tables = append(tables, migrator.Table{
-		Name: "entity_ref",
-		Columns: []*migrator.Column{
-			// Source:
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
-			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: true},
-
-			// Address (defined in the body, not resolved, may be invalid and change)
-			{Name: "family", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: true},
-			{Name: "id", Type: migrator.DB_NVarchar, Length: 1024, Nullable: true},
-
-			// Runtime calcs (will depend on the system state)
-			{Name: "resolved_ok", Type: migrator.DB_Bool, Nullable: false},
-			{Name: "resolved_to", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			{Name: "resolved_warning", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "resolved_time", Type: migrator.DB_DateTime, Nullable: false}, // resolution cache timestamp
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"grn"}, Type: migrator.IndexType},
-			{Cols: []string{"family"}, Type: migrator.IndexType},
-			{Cols: []string{"type"}, Type: migrator.IndexType},
-			{Cols: []string{"resolved_to"}, Type: migrator.IndexType},
-			{Cols: []string{"parent_grn"}, Type: migrator.IndexType},
-		},
-	})
-
-	tables = append(tables, migrator.Table{
-		Name: "entity_history",
-		Columns: []*migrator.Column{
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
-			{Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
-
-			// Raw bytes
-			{Name: "body", Type: migrator.DB_LongBlob, Nullable: false},
-			{Name: "size", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false, IsLatin: true}, // md5(body)
-
-			// Who changed what when
-			{Name: "updated_at", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "updated_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
-
-			// Commit message
-			{Name: "message", Type: migrator.DB_Text, Nullable: false}, // defaults to empty string
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"grn", "version"}, Type: migrator.UniqueIndex},
-			{Cols: []string{"updated_by"}, Type: migrator.IndexType},
-		},
-	})
-
-	tables = append(tables, migrator.Table{
-		Name: "entity_nested",
-		Columns: []*migrator.Column{
-			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false, IsPrimaryKey: true},
-			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
-
-			// The entity identifier
-			{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
-			{Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-			{Name: "folder", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
-
-			// Summary data (always extracted from the `body` column)
-			{Name: "name", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "description", Type: migrator.DB_NVarchar, Length: 255, Nullable: true},
-			{Name: "labels", Type: migrator.DB_Text, Nullable: true}, // JSON object
-			{Name: "fields", Type: migrator.DB_Text, Nullable: true}, // JSON object
-			{Name: "errors", Type: migrator.DB_Text, Nullable: true}, // JSON object
-		},
-		Indices: []*migrator.Index{
-			{Cols: []string{"parent_grn"}},
-			{Cols: []string{"kind"}},
-			{Cols: []string{"folder"}},
-			{Cols: []string{"uid"}},
-			{Cols: []string{"tenant_id", "kind", "uid"}, Type: migrator.UniqueIndex},
-		},
-	})
-
-	// !!! This should not run in production!
-	// The object store SQL schema is still in active development and this
-	// will only be called when the feature toggle is enabled
-	// this check should not be necessary, but is added as an extra check
-	if setting.Env == setting.Prod {
-		return
-	}
-
-	// Migration cleanups: given that this is a complex setup
-	// that requires a lot of testing before we are ready to push out of dev
-	// this script lets us easy wipe previous changes and initialize clean tables
-	suffix := " (v010)" // change this when we want to wipe and reset the object tables
-	mg.AddMigration("EntityStore init: cleanup"+suffix, migrator.NewRawSQLMigration(strings.TrimSpace(`
-		DELETE FROM migration_log WHERE migration_id LIKE 'EntityStore init%';
-	`)))
-	// for a while this was called "ObjectStore"... this can be removed before we remove the dev only flags
-	mg.AddMigration("EntityStore init: object cleanup"+suffix, migrator.NewRawSQLMigration(strings.TrimSpace(`
-		DELETE FROM migration_log WHERE migration_id LIKE 'ObjectStore init%';
-	`)))
-
-	// Initialize all tables
-	for t := range tables {
-		mg.AddMigration("EntityStore init: drop "+tables[t].Name+suffix, migrator.NewRawSQLMigration(
-			fmt.Sprintf("DROP TABLE IF EXISTS %s", tables[t].Name),
-		))
-		mg.AddMigration("EntityStore init: table "+tables[t].Name+suffix, migrator.NewAddTableMigration(tables[t]))
-		for i := range tables[t].Indices {
-			mg.AddMigration(fmt.Sprintf("EntityStore init: index %s[%d]"+suffix, tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i]))
-		}
-	}
-
-	mg.AddMigration("EntityStore init: set path collation in entity tables"+suffix, migrator.NewRawSQLMigration("").
-		// MySQL `utf8mb4_unicode_ci` collation is set in `mysql_dialect.go`
-		// SQLite uses a `BINARY` collation by default
-		Postgres("ALTER TABLE entity_folder ALTER COLUMN slug_path TYPE VARCHAR(1024) COLLATE \"C\";")) // Collate C - sorting done based on character code byte values
-}

pkg/services/sqlstore/migrations/migrations.go

@@ -1,7 +1,6 @@
 package migrations
 
 import (
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrations/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert"
 	. "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
@@ -23,7 +22,7 @@ func ProvideOSSMigrations() *OSSMigrations {
 }
 
 func (*OSSMigrations) AddMigration(mg *Migrator) {
-	addMigrationLogMigrations(mg)
+	mg.AddCreateMigration()
 	addUserMigrations(mg)
 	addTempUserMigrations(mg)
 	addStarMigrations(mg)
@@ -70,12 +69,6 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
 
 	addCorrelationsMigrations(mg)
 
-	if mg.Cfg != nil && mg.Cfg.IsFeatureToggleEnabled != nil {
-		if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagEntityStore) {
-			addEntityStoreMigrations(mg)
-		}
-	}
-
 	addEntityEventsTableMigration(mg)
 
 	addPublicDashboardMigration(mg)
@@ -98,22 +91,6 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
 	addFolderMigrations(mg)
 }
 
-func addMigrationLogMigrations(mg *Migrator) {
-	migrationLogV1 := Table{
-		Name: "migration_log",
-		Columns: []*Column{
-			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
-			{Name: "migration_id", Type: DB_NVarchar, Length: 255},
-			{Name: "sql", Type: DB_Text},
-			{Name: "success", Type: DB_Bool},
-			{Name: "error", Type: DB_Text},
-			{Name: "timestamp", Type: DB_DateTime},
-		},
-	}
-
-	mg.AddMigration("create migration_log table", NewAddTableMigration(migrationLogV1))
-}
-
 func addStarMigrations(mg *Migrator) {
 	starV1 := Table{
 		Name: "star",

pkg/services/sqlstore/migrator/migrator.go

@@ -28,6 +28,7 @@ type Migrator struct {
 	Cfg          *setting.Cfg
 	isLocked     atomic.Bool
 	logMap       map[string]MigrationLog
+	tableName    string
 }
 
 type MigrationLog struct {
@@ -40,16 +41,44 @@ type MigrationLog struct {
 }
 
 func NewMigrator(engine *xorm.Engine, cfg *setting.Cfg) *Migrator {
-	mg := &Migrator{}
-	mg.DBEngine = engine
-	mg.Logger = log.New("migrator")
-	mg.migrations = make([]Migration, 0)
-	mg.migrationIds = make(map[string]struct{})
-	mg.Dialect = NewDialect(mg.DBEngine)
-	mg.Cfg = cfg
+	return NewScopedMigrator(engine, cfg, "")
+}
+
+// NewScopedMigrator should only be used for the transition to a new storage engine
+func NewScopedMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string) *Migrator {
+	mg := &Migrator{
+		Cfg:          cfg,
+		DBEngine:     engine,
+		migrations:   make([]Migration, 0),
+		migrationIds: make(map[string]struct{}),
+		Dialect:      NewDialect(engine),
+	}
+	if scope == "" {
+		mg.tableName = "migration_log"
+		mg.Logger = log.New("migrator")
+	} else {
+		mg.tableName = scope + "_migration_log"
+		mg.Logger = log.New(scope + " migrator")
+	}
 	return mg
 }
 
+// AddCreateMigration adds the initial migration log table -- this should likely be
+// automatic and first, but enough tests exists that do not expect that we can keep it explicit
+func (mg *Migrator) AddCreateMigration() {
+	mg.AddMigration("create "+mg.tableName+" table", NewAddTableMigration(Table{
+		Name: mg.tableName,
+		Columns: []*Column{
+			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "migration_id", Type: DB_NVarchar, Length: 255},
+			{Name: "sql", Type: DB_Text},
+			{Name: "success", Type: DB_Bool},
+			{Name: "error", Type: DB_Text},
+			{Name: "timestamp", Type: DB_DateTime},
+		},
+	}))
+}
+
 func (mg *Migrator) MigrationsCount() int {
 	return len(mg.migrations)
 }
@@ -79,7 +108,7 @@ func (mg *Migrator) GetMigrationLog() (map[string]MigrationLog, error) {
 	logMap := make(map[string]MigrationLog)
 	logItems := make([]MigrationLog, 0)
 
-	exists, err := mg.DBEngine.IsTableExist(new(MigrationLog))
+	exists, err := mg.DBEngine.IsTableExist(mg.tableName)
 	if err != nil {
 		return nil, fmt.Errorf("%v: %w", "failed to check table existence", err)
 	}
@@ -87,7 +116,7 @@ func (mg *Migrator) GetMigrationLog() (map[string]MigrationLog, error) {
 		return logMap, nil
 	}
 
-	if err = mg.DBEngine.Find(&logItems); err != nil {
+	if err = mg.DBEngine.Table(mg.tableName).Find(&logItems); err != nil {
 		return nil, err
 	}
 
@@ -167,7 +196,7 @@ func (mg *Migrator) run() (err error) {
 				mg.Logger.Error("Exec failed", "error", err, "sql", sql)
 				record.Error = err.Error()
 				if !m.SkipMigrationLog() {
-					if _, err := sess.Insert(&record); err != nil {
+					if _, err := sess.Table(mg.tableName).Insert(&record); err != nil {
 						return err
 					}
 				}
@@ -175,7 +204,7 @@ func (mg *Migrator) run() (err error) {
 			}
 			record.Success = true
 			if !m.SkipMigrationLog() {
-				_, err = sess.Insert(&record)
+				_, err = sess.Table(mg.tableName).Insert(&record)
 			}
 			if err == nil {
 				migrationsPerformed++

pkg/services/sqlstore/sqlstore.go

@@ -479,6 +479,10 @@ func (ss *SQLStore) readConfig() error {
 	return nil
 }
 
+func (ss *SQLStore) GetMigrationLockAttemptTimeout() int {
+	return ss.dbCfg.MigrationLockAttemptTimeout
+}
+
 func (ss *SQLStore) RecursiveQueriesAreSupported() (bool, error) {
 	if ss.recursiveQueriesAreSupported != nil {
 		return *ss.recursiveQueriesAreSupported, nil