From 91495440868b61f50af8c0b2724fe2e3b1963e4b Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 31 Oct 2022 08:28:19 -0700 Subject: [PATCH] Store: Add SQL tables for SQL based object store (dev mode only) (#57785) --- .../sqlstore/migrations/migrations.go | 4 + .../sqlstore/migrations/object_store_mig.go | 151 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 pkg/services/sqlstore/migrations/object_store_mig.go diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 2013bbafc05..85bc3dadc95 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -82,6 +82,10 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addCommentGroupMigrations(mg) addCommentMigrations(mg) } + + if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagObjectStore) { + addObjectStorageMigrations(mg) + } } addEntityEventsTableMigration(mg) diff --git a/pkg/services/sqlstore/migrations/object_store_mig.go b/pkg/services/sqlstore/migrations/object_store_mig.go new file mode 100644 index 00000000000..27bea6cbcef --- /dev/null +++ b/pkg/services/sqlstore/migrations/object_store_mig.go @@ -0,0 +1,151 @@ +package migrations + +import ( + "fmt" + "strings" + + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/setting" +) + +func addObjectStorageMigrations(mg *migrator.Migrator) { + tables := []migrator.Table{} + tables = append(tables, migrator.Table{ + Name: "object", + Columns: []*migrator.Column{ + // Object key contains everything required to make it unique across all instances + // orgId+scope+kind+uid + {Name: "key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false, IsPrimaryKey: true}, + + // This is an optimization for listing everything at the same level in the object store + {Name: "parent_folder_key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + + // The object type + {Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + + // The raw object body (any byte array) + {Name: "body", Type: migrator.DB_Blob, Nullable: false}, + {Name: "size", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false}, // md5(body) + {Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false}, + + // Who changed what when -- We should avoid JOINs with other tables in the database + {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "created", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "updated_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, + {Name: "created_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, + + // For objects that are synchronized from an external source (ie provisioning or git) + {Name: "sync_src", Type: migrator.DB_Text, Nullable: true}, + {Name: "sync_time", Type: migrator.DB_DateTime, Nullable: true}, + + // Summary data (always extracted from the `body` column) + {Name: "name", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "description", Type: migrator.DB_NVarchar, Length: 255, Nullable: true}, + {Name: "labels", Type: migrator.DB_Text, Nullable: true}, // JSON object + {Name: "fields", Type: migrator.DB_Text, Nullable: true}, // JSON object + {Name: "errors", Type: migrator.DB_Text, Nullable: true}, // JSON object + }, + PrimaryKeys: []string{"key"}, + Indices: []*migrator.Index{ + {Cols: []string{"parent_folder_key"}}, // list in folder + {Cols: []string{"kind"}}, // filter by type + }, + }) + + tables = append(tables, migrator.Table{ + Name: "object_labels", + Columns: []*migrator.Column{ + {Name: "key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + {Name: "label", Type: migrator.DB_NVarchar, Length: 191, Nullable: false}, + {Name: "value", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"key", "label"}, Type: migrator.UniqueIndex}, + }, + }) + + tables = append(tables, migrator.Table{ + Name: "object_ref", + Columns: []*migrator.Column{ + // Source: + {Name: "key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + + // Address (defined in the body, not resolved, may be invalid and change) + {Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: true}, + {Name: "uid", Type: migrator.DB_NVarchar, Length: 1024, Nullable: true}, + + // Runtime calcs (will depend on the system state) + {Name: "resolved_ok", Type: migrator.DB_Bool, Nullable: false}, + {Name: "resolved_to", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + {Name: "resolved_warning", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "resolved_time", Type: migrator.DB_DateTime, Nullable: false}, // resolution cache timestamp + }, + Indices: []*migrator.Index{ + {Cols: []string{"key"}, Type: migrator.IndexType}, + {Cols: []string{"kind"}, Type: migrator.IndexType}, + {Cols: []string{"resolved_to"}, Type: migrator.IndexType}, + }, + }) + + tables = append(tables, migrator.Table{ + Name: "object_history", + Columns: []*migrator.Column{ + {Name: "key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false}, + {Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false}, + + // Raw bytes + {Name: "body", Type: migrator.DB_Blob, Nullable: false}, + {Name: "size", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false}, // md5(body) + + // Who changed what when + {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "updated_by", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, + + // Commit message + {Name: "message", Type: migrator.DB_Text, Nullable: false}, // defaults to empty string + }, + Indices: []*migrator.Index{ + {Cols: []string{"key", "version"}, Type: migrator.UniqueIndex}, + {Cols: []string{"updated_by"}, Type: migrator.IndexType}, + }, + }) + + // !!! This should not run in production! + // The object store SQL schema is still in active development and this + // will only be called when the feature toggle is enabled + // this check should not be necessary, but is added as an extra check + if setting.Env == setting.Prod { + return + } + + // Migration cleanups: given that this is a complex setup + // that requires a lot of testing before we are ready to push out of dev + // this script lets us easy wipe previous changes and initialize clean tables + suffix := " (v0)" // change this when we want to wipe and reset the object tables + mg.AddMigration("ObjectStore init: cleanup"+suffix, migrator.NewRawSQLMigration(strings.TrimSpace(` + DELETE FROM migration_log WHERE migration_id LIKE 'ObjectStore init%'; + DROP table if exists "object"; + DROP table if exists "object_ref"; + DROP table if exists "object_history"; + DROP table if exists "object_labels"; + DROP table if exists "object_alias"; + DROP table if exists "object_access"; + `))) + + // Initialize all tables + for t := range tables { + mg.AddMigration("ObjectStore init: table "+tables[t].Name+suffix, migrator.NewAddTableMigration(tables[t])) + for i := range tables[t].Indices { + mg.AddMigration(fmt.Sprintf("ObjectStore init: index %s[%d]"+suffix, tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i])) + } + } + + // TODO: add collation support to `migrator.Column` + mg.AddMigration("ObjectStore init: set path collation in object tables"+suffix, migrator.NewRawSQLMigration(""). + // MySQL `utf8mb4_unicode_ci` collation is set in `mysql_dialect.go` + // SQLite uses a `BINARY` collation by default + Postgres("ALTER TABLE object ALTER COLUMN path TYPE VARCHAR(1024) COLLATE \"C\";")) // Collate C - sorting done based on character code byte values +}