[release-11.6.10] Fix tables without PKs (including tables from current main) (#115537)
* Skip empty migrations. * Backport SecretDB to release-11.6, and initialize it on startup. * Add missing file. * Move dependency registration to apis.go * Use migrator from Grafana 12.2. * Add migrations introducing cloud_migration_snapshot_partition table. * Fix import. * Convert unique keys in 3 tables to primary keys (#115421) * Added method for adding migrations for convering unique to primary key. Based on existing migration for `file` table (in `db_file_storage.go`) migrations. * Added better default migration names. Added ability to override migration name. * Use ConvertUniqueKeyToPrimaryKey for cloud_migration_snapshot_partition table. * Convert resource_version UQE to PK. * Convert secret_encrypted_value UQE to PK. * Removed extra test. * Removed testdata. * Remove support for renaming migrations for now. We can bring it in later, when we want to convert existing migrations for file, file_meta and setting tables. * Revert removal of ColumnName to ease backporting, since this field is referenced from enterprise code. * Use quoted identifiers in Postgres statement. * Ignore data_key_id column that was introduced in Grafana 12.3. * Return empty SQL, this is now ignored.
This commit is contained in:
Peter Štibraný
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/iam"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/query"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/secret"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/userstorage"
|
||||
)
|
||||
|
||||
@@ -28,6 +29,7 @@ func ProvideRegistryServiceSink(
|
||||
_ *notifications.NotificationsAPIBuilder,
|
||||
_ *userstorage.UserStorageAPIBuilder,
|
||||
_ *provisioning.APIBuilder,
|
||||
_ *secret.DependencyRegisterer,
|
||||
) *Service {
|
||||
return &Service{}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
package contracts
|
||||
|
||||
var (
|
||||
// The name used to refer to the system keeper
|
||||
SystemKeeperName = "system"
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
package contracts
|
||||
|
||||
import "context"
|
||||
|
||||
// SecretDBMigrator is an interface for running database migrations related to secrets management.
|
||||
type SecretDBMigrator interface {
|
||||
RunMigrations(ctx context.Context, lockDatabase bool) error
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package secret
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
// DependencyRegisterer is set to satisfy wire gen and make sure the `RegisterDependencies` is called.
|
||||
type DependencyRegisterer struct{}
|
||||
|
||||
func RegisterDependencies(cfg *setting.Cfg, secretDBMigrator contracts.SecretDBMigrator) (*DependencyRegisterer, error) {
|
||||
// Some DBs that claim to be MySQL/Postgres-compatible might not support table locking.
|
||||
lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true)
|
||||
|
||||
// This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator.
|
||||
if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil {
|
||||
return nil, fmt.Errorf("running secret database migrations: %w", err)
|
||||
}
|
||||
|
||||
return &DependencyRegisterer{}, nil
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/iam"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/query"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/secret"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/service"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/userstorage"
|
||||
"github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext"
|
||||
@@ -25,6 +26,9 @@ var WireSet = wire.NewSet(
|
||||
wire.Bind(new(datasource.PluginContextWrapper), new(*plugincontext.Provider)),
|
||||
datasource.ProvideDefaultPluginConfigs,
|
||||
|
||||
// Secrets
|
||||
secret.RegisterDependencies,
|
||||
|
||||
// Each must be added here *and* in the ServiceSink above
|
||||
dashboardinternal.RegisterAPIService,
|
||||
dashboardsnapshot.RegisterAPIService,
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/wire"
|
||||
|
||||
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api"
|
||||
"github.com/grafana/grafana/pkg/api/avatar"
|
||||
"github.com/grafana/grafana/pkg/api/routing"
|
||||
@@ -158,6 +159,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/user/userimpl"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
legacydualwrite "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
secretmigrator "github.com/grafana/grafana/pkg/storage/secret/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
unifiedsearch "github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
|
||||
@@ -401,6 +403,7 @@ var wireBasicSet = wire.NewSet(
|
||||
connectors.ProvideOrgRoleMapper,
|
||||
wire.Bind(new(user.Verifier), new(*userimpl.Verifier)),
|
||||
authz.WireSet,
|
||||
secretmigrator.NewWithEngine,
|
||||
// Unified storage
|
||||
resource.ProvideStorageMetrics,
|
||||
// Kubernetes API server
|
||||
|
||||
+15
-3
File diff suppressed because one or more lines are too long
@@ -98,6 +98,15 @@ func addCloudMigrationsMigrations(mg *Migrator) {
|
||||
{Cols: []string{"uid"}, Type: UniqueIndex},
|
||||
},
|
||||
}
|
||||
migrationSnapshotPartitionTable := Table{
|
||||
Name: "cloud_migration_snapshot_partition",
|
||||
Columns: []*Column{
|
||||
{Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false},
|
||||
{Name: "partition_number", Type: DB_Int, Nullable: false},
|
||||
{Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false},
|
||||
{Name: "data", Type: DB_LongBlob, Nullable: false},
|
||||
},
|
||||
}
|
||||
|
||||
addTableReplaceMigrations(mg, migrationTable, migrationSessionTable, 2, map[string]string{
|
||||
"id": "id",
|
||||
@@ -187,4 +196,23 @@ func addCloudMigrationsMigrations(mg *Migrator) {
|
||||
mg.AddMigration("increase resource_uid column length", NewRawSQLMigration("").
|
||||
Mysql("ALTER TABLE cloud_migration_resource MODIFY resource_uid NVARCHAR(255);").
|
||||
Postgres("ALTER TABLE cloud_migration_resource ALTER COLUMN resource_uid TYPE VARCHAR(255);"))
|
||||
|
||||
mg.AddMigration("create cloud_migration_snapshot_partition table v1", NewAddTableMigration(migrationSnapshotPartitionTable))
|
||||
srpUniqueIndex := Index{
|
||||
Name: "srp_unique",
|
||||
Cols: []string{"snapshot_uid", "resource_type", "partition_number"}, Type: UniqueIndex,
|
||||
}
|
||||
mg.AddMigration("add cloud_migration_snapshot_partition srp_unique index", NewAddIndexMigration(migrationSnapshotPartitionTable, &srpUniqueIndex))
|
||||
|
||||
updatedCloudMigrationSnapshotPartitionTable := Table{
|
||||
Name: "cloud_migration_snapshot_partition",
|
||||
Columns: []*Column{
|
||||
{Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "partition_number", Type: DB_Int, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "data", Type: DB_LongBlob, Nullable: false},
|
||||
},
|
||||
PrimaryKeys: []string{"snapshot_uid", "resource_type", "partition_number"},
|
||||
}
|
||||
ConvertUniqueKeyToPrimaryKey(mg, srpUniqueIndex, updatedCloudMigrationSnapshotPartitionTable)
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) {
|
||||
mg.AddMigration("drop file_path unique index from file table if it exists (mysql)", mysqlMigration2)
|
||||
|
||||
mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file ADD PRIMARY KEY (path_hash);`)
|
||||
mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file", ColumnName: "path_hash"}
|
||||
mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file"}
|
||||
mg.AddMigration("add primary key to file table if it doesn't exist (mysql)", mysqlMigration3)
|
||||
|
||||
postgres := `
|
||||
@@ -162,7 +162,7 @@ func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) {
|
||||
mg.AddMigration("drop file_path unique index from file_meta table if it exists (mysql)", mysqlMigration2)
|
||||
|
||||
mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);`)
|
||||
mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta", ColumnName: "path_hash"}
|
||||
mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta"}
|
||||
mg.AddMigration("add primary key to file_meta table if it doesn't exist (mysql)", mysqlMigration3)
|
||||
|
||||
postgres := `
|
||||
|
||||
@@ -241,7 +241,7 @@ func (b *BaseDialect) CopyTableData(sourceTable string, targetTable string, sour
|
||||
targetColsSQL := b.QuoteColList(targetCols)
|
||||
|
||||
quote := b.dialect.Quote
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable))
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)\nSELECT %s\nFROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable))
|
||||
}
|
||||
|
||||
func (b *BaseDialect) DropTable(tableName string) string {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -53,7 +55,7 @@ func (m *RawSQLMigration) SQL(dialect Dialect) string {
|
||||
}
|
||||
}
|
||||
|
||||
return dialect.NoOpSQL()
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *RawSQLMigration) Set(dialect string, sql string) *RawSQLMigration {
|
||||
@@ -275,3 +277,155 @@ func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharset
|
||||
func (m *TableCharsetMigration) SQL(d Dialect) string {
|
||||
return d.UpdateTableSQL(m.tableName, m.columns)
|
||||
}
|
||||
|
||||
type addPrimaryKeyMigration struct {
|
||||
MigrationBase
|
||||
tableName string
|
||||
uniqueKey Index
|
||||
|
||||
// Used for Sqlite recreation of the table. Temporary table will have tableName + "_new" suffix.
|
||||
table Table
|
||||
}
|
||||
|
||||
func (m *addPrimaryKeyMigration) SQL(d Dialect) string {
|
||||
if d.DriverName() == SQLite {
|
||||
// Final SQL will do following in the individual statements:
|
||||
// 1. Create new temporary table
|
||||
// 2. Copy data from old table to temporary table
|
||||
// 3. Drop old table, rename temporary table to original name
|
||||
// 4. Recreate indexes for table.
|
||||
//
|
||||
// For example:
|
||||
//
|
||||
// CREATE TABLE file_new
|
||||
// (
|
||||
// path TEXT NOT NULL,
|
||||
// path_hash TEXT NOT NULL,
|
||||
// parent_folder_path_hash TEXT NOT NULL,
|
||||
// contents BLOB NOT NULL,
|
||||
// etag TEXT NOT NULL,
|
||||
// cache_control TEXT NOT NULL,
|
||||
// content_disposition TEXT NOT NULL,
|
||||
// updated DATETIME NOT NULL,
|
||||
// created DATETIME NOT NULL,
|
||||
// size INTEGER NOT NULL,
|
||||
// mime_type TEXT NOT NULL,
|
||||
//
|
||||
// PRIMARY KEY (path_hash)
|
||||
// );
|
||||
//
|
||||
// INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type)
|
||||
// SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file;
|
||||
//
|
||||
// DROP TABLE file;
|
||||
// ALTER TABLE file_new RENAME TO file;
|
||||
//
|
||||
// CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash);
|
||||
|
||||
tempTable := m.table
|
||||
tempTable.Name = m.tableName + "_new"
|
||||
|
||||
statements := strings.Builder{}
|
||||
|
||||
statements.WriteString(d.CreateTableSQL(&tempTable))
|
||||
statements.WriteString("\n") // CreateTableSQL adds semicolon
|
||||
|
||||
cols := make([]string, 0, len(tempTable.Columns))
|
||||
for _, col := range tempTable.Columns {
|
||||
cols = append(cols, col.Name)
|
||||
}
|
||||
statements.WriteString(d.CopyTableData(m.tableName, tempTable.Name, cols, cols))
|
||||
statements.WriteString(";\n")
|
||||
|
||||
statements.WriteString(d.DropTable(m.tableName))
|
||||
statements.WriteString(";\n")
|
||||
|
||||
statements.WriteString(d.RenameTable(tempTable.Name, m.tableName))
|
||||
statements.WriteString(";\n")
|
||||
|
||||
for _, idx := range tempTable.Indices {
|
||||
// Use real table name, not temporary one now
|
||||
statements.WriteString(d.CreateIndexSQL(m.tableName, idx))
|
||||
statements.WriteString("\n") // CreateIndexSQL adds semicolon
|
||||
}
|
||||
|
||||
return statements.String()
|
||||
} else if d.DriverName() == Postgres {
|
||||
quotesCols := make([]string, 0, len(m.uniqueKey.Cols))
|
||||
for _, c := range m.uniqueKey.Cols {
|
||||
quotesCols = append(quotesCols, d.Quote(c))
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Drop the unique constraint if it exists
|
||||
DROP INDEX IF EXISTS %s;
|
||||
|
||||
-- Add primary key if it doesn't already exist
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = '%s'::regclass AND indisprimary) THEN
|
||||
ALTER TABLE %s ADD PRIMARY KEY (%s);
|
||||
END IF;
|
||||
END $$;`, d.Quote(m.uniqueKey.XName(m.tableName)), m.tableName, d.Quote(m.tableName), strings.Join(quotesCols, ","))
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ConvertUniqueKeyToPrimaryKey adds series of migrations to convert existing unique key to PRIMARY KEY.
|
||||
// For Sqlite this means recreating the table, which only works if there are no foreign keys referencing the table.
|
||||
func ConvertUniqueKeyToPrimaryKey(mg *Migrator, uniqueKey Index, finalTable Table) {
|
||||
tableName := finalTable.Name
|
||||
if tableName == "" {
|
||||
panic("invalid table name")
|
||||
}
|
||||
if len(uniqueKey.Cols) == 0 || uniqueKey.Type != UniqueIndex {
|
||||
panic("invalid unique type")
|
||||
}
|
||||
if !slices.Equal(uniqueKey.Cols, finalTable.PrimaryKeys) {
|
||||
panic("invalid primary key in the final table")
|
||||
}
|
||||
|
||||
colPks := map[string]bool{}
|
||||
for _, col := range finalTable.Columns {
|
||||
if col.IsPrimaryKey {
|
||||
colPks[col.Name] = true
|
||||
}
|
||||
}
|
||||
for _, c := range uniqueKey.Cols {
|
||||
if !colPks[c] {
|
||||
panic(fmt.Sprintf("column %s is not part of primary key in the table definition", c))
|
||||
}
|
||||
}
|
||||
|
||||
columnsList := strings.Join(uniqueKey.Cols, ",")
|
||||
|
||||
mysqlQuote := NewDialect(MySQL).Quote
|
||||
mysqlQuotedColumns := make([]string, 0, len(uniqueKey.Cols))
|
||||
for _, col := range uniqueKey.Cols {
|
||||
mysqlQuotedColumns = append(mysqlQuotedColumns, mysqlQuote(col))
|
||||
}
|
||||
|
||||
// migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON
|
||||
// in this case we need to do the conversion in one sql statement
|
||||
mysqlMigration1 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`
|
||||
ALTER TABLE %s
|
||||
DROP PRIMARY KEY,
|
||||
DROP COLUMN my_row_id,
|
||||
DROP INDEX %s,
|
||||
ADD PRIMARY KEY (%s);
|
||||
`, tableName, uniqueKey.XName(tableName), strings.Join(mysqlQuotedColumns, ",")))
|
||||
mysqlMigration1.Condition = &IfColumnExistsCondition{TableName: tableName, ColumnName: "my_row_id"}
|
||||
mg.AddMigration(fmt.Sprintf("drop my_row_id and add primary key with columns %s to table %s if my_row_id exists (auto-generated mysql column)", columnsList, tableName), mysqlMigration1)
|
||||
|
||||
mysqlMigration2 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s DROP INDEX %s`, tableName, uniqueKey.XName(tableName)))
|
||||
mysqlMigration2.Condition = &IfIndexExistsCondition{TableName: tableName, IndexName: uniqueKey.XName(tableName)}
|
||||
mg.AddMigration(fmt.Sprintf("drop unique index %s from %s table if it exists (mysql)", uniqueKey.XName(tableName), tableName), mysqlMigration2)
|
||||
|
||||
mysqlMigration3 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s ADD PRIMARY KEY (%s)`, tableName, strings.Join(mysqlQuotedColumns, ",")))
|
||||
mysqlMigration3.Condition = &IfPrimaryKeyNotExistsCondition{TableName: tableName}
|
||||
mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s if it doesn't exist (mysql)", columnsList, tableName), mysqlMigration3)
|
||||
|
||||
// postgres and sqlite statements are idempotent so we can have only one condition-less migration
|
||||
mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s (postgres and sqlite)", columnsList, tableName), &addPrimaryKeyMigration{tableName: tableName, uniqueKey: uniqueKey, table: finalTable})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
//go:embed testdata/sqlite_file_migration_statement.sql
|
||||
var sqliteMigrationStatement string
|
||||
|
||||
func TestConvertUniqueKeyToPrimaryKey(t *testing.T) {
|
||||
names := []string{
|
||||
"drop my_row_id and add primary key with columns path_hash,etag to table file if my_row_id exists (auto-generated mysql column)",
|
||||
"drop unique index UQE_file_path_hash_etag from file table if it exists (mysql)",
|
||||
"add primary key with columns path_hash,etag to table file if it doesn't exist (mysql)",
|
||||
"add primary key with columns path_hash,etag to table file (postgres and sqlite)",
|
||||
}
|
||||
expectedMigrations := map[string][]ExpectedMigration{
|
||||
MySQL: {
|
||||
{Id: names[0], SQL: `
|
||||
ALTER TABLE file
|
||||
DROP PRIMARY KEY,
|
||||
DROP COLUMN my_row_id,
|
||||
DROP INDEX UQE_file_path_hash_etag,
|
||||
ADD PRIMARY KEY (` + "`path_hash`" + `,` + "`etag`" + `);`},
|
||||
{Id: names[1], SQL: "ALTER TABLE file DROP INDEX UQE_file_path_hash_etag"},
|
||||
{Id: names[2], SQL: "ALTER TABLE file ADD PRIMARY KEY (`path_hash`,`etag`)"},
|
||||
{Id: names[3], SQL: ""},
|
||||
},
|
||||
Postgres: {
|
||||
{Id: names[0], SQL: ""},
|
||||
{Id: names[1], SQL: ""},
|
||||
{Id: names[2], SQL: ""},
|
||||
{Id: names[3], SQL: `
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Drop the unique constraint if it exists
|
||||
DROP INDEX IF EXISTS "UQE_file_path_hash_etag";
|
||||
|
||||
-- Add primary key if it doesn't already exist
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN
|
||||
ALTER TABLE "file" ADD PRIMARY KEY ("path_hash","etag");
|
||||
END IF;
|
||||
END $$;`},
|
||||
},
|
||||
SQLite: {
|
||||
{Id: names[0], SQL: ""},
|
||||
{Id: names[1], SQL: ""},
|
||||
{Id: names[2], SQL: ""},
|
||||
{Id: names[3], SQL: sqliteMigrationStatement}, // Embed used here because sqlite statement is full of backquotes.
|
||||
},
|
||||
}
|
||||
|
||||
for dialectName, migrations := range expectedMigrations {
|
||||
t.Run(dialectName, func(t *testing.T) {
|
||||
err := CheckExpectedMigrations(dialectName, migrations, func(migrator *Migrator) {
|
||||
ConvertUniqueKeyToPrimaryKey(migrator,
|
||||
Index{Cols: []string{"path_hash", "etag"}, Type: UniqueIndex}, // Convert this unique key to primary key
|
||||
Table{
|
||||
Name: "file",
|
||||
Columns: []*Column{
|
||||
{Name: "path", Type: DB_NVarchar, Length: 1024, Nullable: false},
|
||||
{Name: "path_hash", Type: DB_NVarchar, Length: 64, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "parent_folder_path_hash", Type: DB_NVarchar, Length: 64, Nullable: false},
|
||||
{Name: "contents", Type: DB_Blob, Nullable: false},
|
||||
{Name: "etag", Type: DB_NVarchar, Length: 32, Nullable: false, IsPrimaryKey: true},
|
||||
},
|
||||
PrimaryKeys: []string{"path_hash", "etag"},
|
||||
Indices: []*Index{
|
||||
{Cols: []string{"parent_folder_path_hash"}},
|
||||
},
|
||||
})
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
@@ -67,12 +68,16 @@ func NewMigrator(engine *xorm.Engine, cfg *setting.Cfg) *Migrator {
|
||||
|
||||
// NewScopedMigrator should only be used for the transition to a new storage engine
|
||||
func NewScopedMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string) *Migrator {
|
||||
return newMigrator(engine, cfg, scope, NewDialect(engine.DriverName()))
|
||||
}
|
||||
|
||||
func newMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string, dialect Dialect) *Migrator {
|
||||
mg := &Migrator{
|
||||
Cfg: cfg,
|
||||
DBEngine: engine,
|
||||
migrations: make([]Migration, 0),
|
||||
migrationIds: make(map[string]struct{}),
|
||||
Dialect: NewDialect(engine.DriverName()),
|
||||
Dialect: dialect,
|
||||
metrics: migratorMetrics{
|
||||
migCount: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "grafana_database",
|
||||
@@ -392,8 +397,12 @@ func (mg *Migrator) exec(ctx context.Context, m Migration, sess *xorm.Session) e
|
||||
err = codeMigration.Exec(sess, mg)
|
||||
} else {
|
||||
sql := m.SQL(mg.Dialect)
|
||||
logger.Debug("Executing sql migration", "id", m.Id(), "sql", sql)
|
||||
_, err = sess.Exec(sql)
|
||||
if strings.TrimSpace(sql) == "" {
|
||||
logger.Debug("Skipping empty sql migration", "id", m.Id())
|
||||
} else {
|
||||
logger.Debug("Executing sql migration", "id", m.Id(), "sql", sql)
|
||||
_, err = sess.Exec(sql)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE IF NOT EXISTS `file_new` (
|
||||
`path` TEXT NOT NULL
|
||||
, `path_hash` TEXT NOT NULL
|
||||
, `parent_folder_path_hash` TEXT NOT NULL
|
||||
, `contents` BLOB NOT NULL
|
||||
, `etag` TEXT NOT NULL
|
||||
, PRIMARY KEY ( `path_hash`,`etag` ));
|
||||
|
||||
INSERT INTO `file_new` (`path`
|
||||
, `path_hash`
|
||||
, `parent_folder_path_hash`
|
||||
, `contents`
|
||||
, `etag`)
|
||||
SELECT `path`
|
||||
, `path_hash`
|
||||
, `parent_folder_path_hash`
|
||||
, `contents`
|
||||
, `etag`
|
||||
FROM `file`;
|
||||
|
||||
DROP TABLE IF EXISTS `file`;
|
||||
ALTER TABLE `file_new` RENAME TO `file`;
|
||||
CREATE INDEX `IDX_file_parent_folder_path_hash` ON `file` (`parent_folder_path_hash`);
|
||||
@@ -0,0 +1,51 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ExpectedMigration struct {
|
||||
Id string
|
||||
SQL string
|
||||
}
|
||||
|
||||
// CheckExpectedMigrations verifies that given migrations exist in migrator after running addMigrations function,
|
||||
// that they are in the same order and have expected SQL.
|
||||
func CheckExpectedMigrations(dialectName string, expected []ExpectedMigration, addMigrations func(migrator *Migrator)) error {
|
||||
d := NewDialect(dialectName)
|
||||
mg := newMigrator(nil, nil, "", d)
|
||||
addMigrations(mg)
|
||||
|
||||
migrations := mg.migrations
|
||||
migrationNames := make([]string, 0, len(migrations))
|
||||
for _, m := range expected {
|
||||
for ; len(migrations) > 0 && migrations[0].Id() != m.Id; migrations = migrations[1:] {
|
||||
migrationNames = append(migrationNames, migrations[0].Id())
|
||||
}
|
||||
|
||||
if len(migrations) == 0 {
|
||||
return fmt.Errorf("migration `%s` not found, existing migrations:\n%s", m.Id, strings.Join(migrationNames, "\n"))
|
||||
}
|
||||
|
||||
sql := migrations[0].SQL(d)
|
||||
if normalizeLines(m.SQL) != normalizeLines(sql) {
|
||||
return fmt.Errorf("migration `%s` has wrong SQL:\nexpected:\n%s\nactual:\n%s", m.Id, m.SQL, sql)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeLines(sql string) string {
|
||||
lines := strings.Split(sql, "\n")
|
||||
result := strings.Builder{}
|
||||
for _, l := range lines {
|
||||
l := strings.TrimSpace(l)
|
||||
if l == "" {
|
||||
continue
|
||||
}
|
||||
result.WriteString(l)
|
||||
result.WriteString("\n")
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
const (
|
||||
TableNameKeeper = "secret_keeper"
|
||||
TableNameSecureValue = "secret_secure_value"
|
||||
TableNameDataKey = "secret_data_key"
|
||||
TableNameEncryptedValue = "secret_encrypted_value"
|
||||
)
|
||||
|
||||
type SecretDB struct {
|
||||
engine *xorm.Engine
|
||||
}
|
||||
|
||||
func New() registry.DatabaseMigrator {
|
||||
return &SecretDB{}
|
||||
}
|
||||
|
||||
func NewWithEngine(db db.DB) contracts.SecretDBMigrator {
|
||||
return &SecretDB{engine: db.GetEngine()}
|
||||
}
|
||||
|
||||
func (db *SecretDB) RunMigrations(ctx context.Context, lockDatabase bool) error {
|
||||
mg := migrator.NewScopedMigrator(db.engine, nil, "secret")
|
||||
|
||||
db.AddMigration(mg)
|
||||
|
||||
return mg.RunMigrations(ctx, lockDatabase, 0)
|
||||
}
|
||||
|
||||
func (*SecretDB) AddMigration(mg *migrator.Migrator) {
|
||||
mg.AddCreateMigration()
|
||||
|
||||
mg.AddMigration("Initialize secrets tables", &migrator.RawSQLMigration{})
|
||||
|
||||
tables := []migrator.Table{}
|
||||
|
||||
secureValueTable := migrator.Table{
|
||||
Name: TableNameSecureValue,
|
||||
Columns: []*migrator.Column{
|
||||
// Kubernetes Metadata
|
||||
{Name: "guid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID.
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "annotations", Type: migrator.DB_Text, Nullable: true},
|
||||
{Name: "labels", Type: migrator.DB_Text, Nullable: true},
|
||||
{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
|
||||
{Name: "created_by", Type: migrator.DB_Text, Nullable: false},
|
||||
{Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, // Used as RV (ResourceVersion)
|
||||
{Name: "updated_by", Type: migrator.DB_Text, Nullable: false},
|
||||
|
||||
// Kubernetes Status
|
||||
{Name: "external_id", Type: migrator.DB_Text, Nullable: false},
|
||||
{Name: "active", Type: migrator.DB_Bool, Nullable: false},
|
||||
{Name: "version", Type: migrator.DB_BigInt, Nullable: false},
|
||||
|
||||
// Spec
|
||||
{Name: "description", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Chosen arbitrarily, but should be enough.
|
||||
{Name: "keeper", Type: migrator.DB_NVarchar, Length: 253, Nullable: true}, // Keeper name, if not set, use default keeper.
|
||||
{Name: "decrypters", Type: migrator.DB_Text, Nullable: true},
|
||||
{Name: "ref", Type: migrator.DB_NVarchar, Length: 1024, Nullable: true}, // Reference to third-party storage secret path.Chosen arbitrarily, but should be enough.
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{Cols: []string{"namespace", "name", "version", "active"}, Type: migrator.UniqueIndex},
|
||||
{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex},
|
||||
},
|
||||
}
|
||||
tables = append(tables, secureValueTable)
|
||||
|
||||
tables = append(tables, migrator.Table{
|
||||
Name: TableNameKeeper,
|
||||
Columns: []*migrator.Column{
|
||||
// Kubernetes Metadata
|
||||
{Name: "guid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID.
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "annotations", Type: migrator.DB_Text, Nullable: true},
|
||||
{Name: "labels", Type: migrator.DB_Text, Nullable: true},
|
||||
{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
|
||||
{Name: "created_by", Type: migrator.DB_Text, Nullable: false},
|
||||
{Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, // Used as RV (ResourceVersion)
|
||||
{Name: "updated_by", Type: migrator.DB_Text, Nullable: false},
|
||||
|
||||
// Spec
|
||||
{Name: "description", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Chosen arbitrarily, but should be enough.
|
||||
{Name: "type", Type: migrator.DB_Text, Nullable: false},
|
||||
// Each keeper has a different payload so we store the whole thing as a blob.
|
||||
{Name: "payload", Type: migrator.DB_Text, Nullable: true},
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{Cols: []string{"namespace", "name"}, Type: migrator.UniqueIndex},
|
||||
},
|
||||
})
|
||||
|
||||
dataKeyTable := migrator.Table{
|
||||
Name: TableNameDataKey,
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true}, // Arbitrarily chosen.
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false}, // Arbitrarily chosen.
|
||||
{Name: "active", Type: migrator.DB_Bool, Nullable: false},
|
||||
{Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false}, // Arbitrarily chosen.
|
||||
{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
|
||||
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
|
||||
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
|
||||
},
|
||||
Indices: []*migrator.Index{},
|
||||
}
|
||||
tables = append(tables, dataKeyTable)
|
||||
|
||||
encryptedValueTable := migrator.Table{
|
||||
Name: TableNameEncryptedValue,
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false},
|
||||
{Name: "version", Type: migrator.DB_BigInt, Nullable: false},
|
||||
{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
|
||||
{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
|
||||
{Name: "updated", Type: migrator.DB_BigInt, Nullable: false},
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex},
|
||||
},
|
||||
}
|
||||
tables = append(tables, encryptedValueTable)
|
||||
|
||||
// Initialize all tables
|
||||
for t := range tables {
|
||||
mg.AddMigration("drop table "+tables[t].Name, migrator.NewDropTableMigration(tables[t].Name))
|
||||
mg.AddMigration("create table "+tables[t].Name, migrator.NewAddTableMigration(tables[t]))
|
||||
for i := range tables[t].Indices {
|
||||
mg.AddMigration(fmt.Sprintf("create table %s, index: %d", tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i]))
|
||||
}
|
||||
}
|
||||
|
||||
mg.AddMigration("create index for list on "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
|
||||
Cols: []string{"namespace", "active", "updated"},
|
||||
Type: migrator.IndexType,
|
||||
}))
|
||||
|
||||
mg.AddMigration("create index for list and read current on "+TableNameDataKey, migrator.NewAddIndexMigration(dataKeyTable, &migrator.Index{
|
||||
Cols: []string{"namespace", "label", "active"},
|
||||
Type: migrator.IndexType,
|
||||
}))
|
||||
|
||||
// Owner Reference columns
|
||||
mg.AddMigration("add owner_reference_api_group column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "owner_reference_api_group",
|
||||
Type: migrator.DB_NVarchar,
|
||||
Length: 253, // Limit enforced by K8s.
|
||||
Nullable: true,
|
||||
}))
|
||||
|
||||
mg.AddMigration("add owner_reference_api_version column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "owner_reference_api_version",
|
||||
Type: migrator.DB_NVarchar,
|
||||
Length: 253, // Limit enforced by K8s.
|
||||
Nullable: true,
|
||||
}))
|
||||
|
||||
mg.AddMigration("add owner_reference_kind column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "owner_reference_kind",
|
||||
Type: migrator.DB_NVarchar,
|
||||
Length: 253, // Limit enforced by K8s.
|
||||
Nullable: true,
|
||||
}))
|
||||
|
||||
mg.AddMigration("add owner_reference_name column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "owner_reference_name",
|
||||
Type: migrator.DB_NVarchar,
|
||||
Length: 253, // Limit enforced by K8s.
|
||||
Nullable: true,
|
||||
}))
|
||||
|
||||
mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "lease_token",
|
||||
Type: migrator.DB_NVarchar,
|
||||
Length: 36,
|
||||
Nullable: true,
|
||||
}))
|
||||
mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
|
||||
Cols: []string{"lease_token"},
|
||||
}))
|
||||
mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
|
||||
Name: "lease_created",
|
||||
Type: migrator.DB_BigInt,
|
||||
Nullable: false,
|
||||
Default: "0",
|
||||
}))
|
||||
mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
|
||||
Cols: []string{"lease_created"},
|
||||
}))
|
||||
|
||||
encryptedValueTableUniqueKey := migrator.Index{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}
|
||||
updatedEncryptedValueTable := migrator.Table{
|
||||
Name: TableNameEncryptedValue,
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, // Limit enforced by K8s.
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "version", Type: migrator.DB_BigInt, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
|
||||
{Name: "created", Type: migrator.DB_BigInt, Nullable: false},
|
||||
{Name: "updated", Type: migrator.DB_BigInt, Nullable: false},
|
||||
// {Name: "data_key_id", Type: migrator.DB_NVarchar, Length: 100, Nullable: false, Default: "''"}, // TODO: Not present until Grafana 12.3.
|
||||
},
|
||||
PrimaryKeys: []string{"namespace", "name", "version"},
|
||||
// TODO: Not present until Grafana 12.3
|
||||
//Indices: []*migrator.Index{
|
||||
// {Cols: []string{"data_key_id"}},
|
||||
//},
|
||||
}
|
||||
migrator.ConvertUniqueKeyToPrimaryKey(mg, encryptedValueTableUniqueKey, updatedEncryptedValueTable)
|
||||
|
||||
}
|
||||
@@ -151,5 +151,17 @@ func initResourceTables(mg *migrator.Migrator) string {
|
||||
|
||||
mg.AddMigration("Migrate DeletionMarkers to real Resource objects", &deletionMarkerMigrator{})
|
||||
|
||||
oldResourceVersionUniqueKey := migrator.Index{Cols: []string{"group", "resource"}, Type: migrator.UniqueIndex}
|
||||
updatedResourceVersionTable := migrator.Table{
|
||||
Name: "resource_version",
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "resource_version", Type: migrator.DB_BigInt, Nullable: false},
|
||||
},
|
||||
PrimaryKeys: []string{"group", "resource"},
|
||||
}
|
||||
|
||||
migrator.ConvertUniqueKeyToPrimaryKey(mg, oldResourceVersionUniqueKey, updatedResourceVersionTable)
|
||||
return marker
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user