feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
Mustafa Sencer Özcan
co-authored by
Rafael Paulovic
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -0,0 +1,12 @@
|
||||
package contract
|
||||
|
||||
import (
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
)
|
||||
|
||||
// UnifiedStorageMigrationService provides unified storage migrations as a background service.
|
||||
// This interface is defined in a separate package to avoid import cycles between
|
||||
// the migrations implementation and packages that need to depend on it (like dualwrite).
|
||||
type UnifiedStorageMigrationService interface {
|
||||
registry.BackgroundService
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
FoldersAndDashboardsMigrationID = "folders and dashboards migration"
|
||||
UnifiedStorageDataMigrationSQL = "unified storage data migration"
|
||||
)
|
||||
|
||||
type dashboardAndFolderMigration struct {
|
||||
migrator.MigrationBase
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
bulkStoreClient resource.ResourceClient
|
||||
}
|
||||
|
||||
var _ migrator.CodeMigration = (*dashboardAndFolderMigration)(nil)
|
||||
|
||||
// SQL implements migrator.Migration interface. Returns a description string.
|
||||
func (sp *dashboardAndFolderMigration) SQL(dialect migrator.Dialect) string {
|
||||
return UnifiedStorageDataMigrationSQL
|
||||
}
|
||||
|
||||
func (sp *dashboardAndFolderMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
|
||||
ctx := context.Background()
|
||||
logger := mg.Logger
|
||||
|
||||
resources := []schema.GroupResource{
|
||||
{
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
},
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
},
|
||||
}
|
||||
|
||||
storageMigrator := newUnifiedStorageMigrator(sp.legacyMigrator, sp.bulkStoreClient, resources, "unified-storage-migration.folders-dashboards")
|
||||
|
||||
orgs, err := sp.getAllOrgs(sess)
|
||||
if err != nil {
|
||||
logger.Error("failed to get organizations for folders and dashboards migration", "error", err)
|
||||
return fmt.Errorf("failed to get organizations: %w", err)
|
||||
}
|
||||
|
||||
if len(orgs) == 0 {
|
||||
logger.Info("No organizations found to migrate, skipping migration")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("Starting migration for all organizations", "org_count", len(orgs))
|
||||
|
||||
for _, org := range orgs {
|
||||
namespace := types.OrgNamespaceFormatter(org.ID)
|
||||
logger.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
|
||||
|
||||
// Create a service identity context for this namespace to authenticate with unified storage
|
||||
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
|
||||
|
||||
if err := storageMigrator.executeMigration(migrationCtx, sess, mg, namespace); err != nil {
|
||||
logger.Error("migration failed for organization", "org_id", org.ID, "org_name", org.Name, "error", err)
|
||||
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
|
||||
return nil
|
||||
}
|
||||
|
||||
type orgInfo struct {
|
||||
ID int64 `xorm:"id"`
|
||||
Name string `xorm:"name"`
|
||||
}
|
||||
|
||||
func (sp *dashboardAndFolderMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
|
||||
var orgs []orgInfo
|
||||
err := sess.Table("org").Cols("id", "name").Find(&orgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orgs, nil
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
|
||||
var logger = log.New("storage.unified.migrations")
|
||||
|
||||
// UnifiedStorageMigrationProvider provides unified storage migrations as a background service
|
||||
type UnifiedStorageMigrationProvider interface {
|
||||
registry.BackgroundService
|
||||
}
|
||||
|
||||
type UnifiedStorageMigrationProviderImpl struct {
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
cfg *setting.Cfg
|
||||
client resource.ResourceClient
|
||||
sqlStore db.DB
|
||||
}
|
||||
|
||||
var _ UnifiedStorageMigrationProvider = (*UnifiedStorageMigrationProviderImpl)(nil)
|
||||
|
||||
// ProvideUnifiedStorageMigrationProvider is a Wire provider that creates the migration service.
|
||||
// The service implements registry.BackgroundService and runs migrations during server startup.
|
||||
func ProvideUnifiedStorageMigrationProvider(
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
cfg *setting.Cfg,
|
||||
client resource.ResourceClient,
|
||||
sqlStore db.DB,
|
||||
) *UnifiedStorageMigrationProviderImpl {
|
||||
return &UnifiedStorageMigrationProviderImpl{
|
||||
legacyMigrator: legacyMigrator,
|
||||
cfg: cfg,
|
||||
client: client,
|
||||
sqlStore: sqlStore,
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes unified storage migrations as a background service.
|
||||
// This blocks until migrations complete. If migrations fail, an error is returned
|
||||
// which will prevent Grafana from starting.
|
||||
func (p *UnifiedStorageMigrationProviderImpl) Run(ctx context.Context) error {
|
||||
// skip migrations in test environments to prevent integration test timeouts.
|
||||
if os.Getenv("GRAFANA_TEST_DB") != "" {
|
||||
return nil
|
||||
}
|
||||
// skip migrations if disabled in config
|
||||
if p.cfg.DisableDataMigrations {
|
||||
logger.Info("Data migrations are disabled, skipping")
|
||||
return nil
|
||||
}
|
||||
// TODO: Re-enable once migrations are ready
|
||||
// return RegisterMigrations(p.legacyMigrator, p.cfg, p.client, p.sqlStore)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterMigrations initializes and registers all unified storage migrations.
|
||||
// This function is the entry point for all data migrations from legacy storage
|
||||
// to unified storage. It returns an error if migrations fail, preventing Grafana
|
||||
// from starting with inconsistent data.
|
||||
func RegisterMigrations(
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
cfg *setting.Cfg,
|
||||
client resource.ResourceClient,
|
||||
sqlStore db.DB,
|
||||
) error {
|
||||
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
|
||||
defer span.End()
|
||||
mg := migrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unified_storage")
|
||||
mg.AddCreateMigration()
|
||||
|
||||
if err := prometheus.Register(mg); err != nil {
|
||||
logger.Warn("Failed to register migrator metrics", "error", err)
|
||||
}
|
||||
|
||||
// Add new migration registrations here for each resource type
|
||||
registerDashboardAndFolderMigration(mg, legacyMigrator, client)
|
||||
|
||||
// Run all registered migrations (blocking)
|
||||
sec := cfg.Raw.Section("database")
|
||||
if err := mg.RunMigrations(ctx, sec.Key("migration_locking").MustBool(true), sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
|
||||
return fmt.Errorf("unified storage data migration failed: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("Unified storage migrations completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func registerDashboardAndFolderMigration(
|
||||
mg *migrator.Migrator,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
bulkStoreClient resource.ResourceClient,
|
||||
) {
|
||||
migration := &dashboardAndFolderMigration{
|
||||
legacyMigrator: legacyMigrator,
|
||||
bulkStoreClient: bulkStoreClient,
|
||||
}
|
||||
mg.AddMigration(FoldersAndDashboardsMigrationID, migration)
|
||||
}
|
||||
@@ -3,164 +3,200 @@ package migrations
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// StorageMigrator defines the interface for executing unified storage migrations
|
||||
type StorageMigrator interface {
|
||||
executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error
|
||||
// Read from legacy and write into unified storage
|
||||
//
|
||||
//go:generate mockery --name UnifiedMigrator --structname MockUnifiedMigrator --inpackage --filename migrator_mock.go --with-expecter
|
||||
type UnifiedMigrator interface {
|
||||
Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
}
|
||||
|
||||
type unifiedStorageMigrator struct {
|
||||
migrator legacy.LegacyMigrator
|
||||
bulkStoreClient resource.ResourceClient
|
||||
resources []schema.GroupResource
|
||||
log log.Logger
|
||||
// unifiedMigration handles the migration of legacy resources to unified storage
|
||||
type unifiedMigration struct {
|
||||
legacy.MigrationDashboardAccessor
|
||||
streamProvider streamProvider
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func newUnifiedStorageMigrator(migrator legacy.LegacyMigrator, bulkStoreClient resource.ResourceClient, resources []schema.GroupResource, logPrefix string) StorageMigrator {
|
||||
return &unifiedStorageMigrator{
|
||||
migrator: migrator,
|
||||
bulkStoreClient: bulkStoreClient,
|
||||
resources: resources,
|
||||
log: log.New(logPrefix),
|
||||
}
|
||||
// streamProvider abstracts the different ways to create a bulk process stream
|
||||
type streamProvider interface {
|
||||
createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error)
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error {
|
||||
startTime := time.Now()
|
||||
m.log.Info("Starting unified storage migration", "namespace", namespace, "resources", m.resources)
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Store: m.bulkStoreClient,
|
||||
LargeObjects: nil, // Not using large object support to avoid import cycles
|
||||
Resources: m.resources,
|
||||
WithHistory: true, // Migrate with full history
|
||||
OnlyCount: false,
|
||||
Progress: func(count int, msg string) {
|
||||
m.log.Info("Migration progress", "count", count, "message", msg)
|
||||
},
|
||||
}
|
||||
|
||||
// Execute the migration via legacy migrator
|
||||
response, err := m.migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
m.log.Error("Migration failed", "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("failed to migrate resources: %w", err)
|
||||
}
|
||||
|
||||
// Validate the migration results
|
||||
if err := m.validateMigration(sess, response); err != nil {
|
||||
m.log.Error("Migration validation failed", "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration validation failed: %w", err)
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed successfully",
|
||||
"duration", time.Since(startTime),
|
||||
"processed", response.Processed,
|
||||
"summaries", len(response.Summary),
|
||||
"rejected", len(response.Rejected))
|
||||
|
||||
return nil
|
||||
// resourceClientStreamProvider creates streams using resource.ResourceClient
|
||||
type resourceClientStreamProvider struct {
|
||||
client resource.ResourceClient
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
|
||||
// Check for rejected items
|
||||
if len(response.Rejected) > 0 {
|
||||
m.log.Warn("Migration had rejected items", "count", len(response.Rejected))
|
||||
for i, rejected := range response.Rejected {
|
||||
if i < 10 { // Log first 10 rejected items
|
||||
m.log.Warn("Rejected item",
|
||||
"namespace", rejected.Key.Namespace,
|
||||
"group", rejected.Key.Group,
|
||||
"resource", rejected.Key.Resource,
|
||||
"name", rejected.Key.Name,
|
||||
"reason", rejected.Error)
|
||||
}
|
||||
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
// Build collection settings for resource client
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.DASHBOARD_RESOURCE,
|
||||
})
|
||||
}
|
||||
// Rejections are not fatal - they may be expected for invalid data
|
||||
}
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
return r.client.BulkProcess(ctx)
|
||||
}
|
||||
|
||||
// bulkStoreClientStreamProvider creates streams using resourcepb.BulkStoreClient
|
||||
type bulkStoreClientStreamProvider struct {
|
||||
client resourcepb.BulkStoreClient
|
||||
}
|
||||
|
||||
func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
// Build collection settings for resource client
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.DASHBOARD_RESOURCE,
|
||||
})
|
||||
}
|
||||
}
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
return b.client.BulkProcess(ctx)
|
||||
}
|
||||
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideUnifiedMigrator(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
client resource.ResourceClient,
|
||||
) UnifiedMigrator {
|
||||
return newUnifiedMigrator(
|
||||
dashboardAccess,
|
||||
&resourceClientStreamProvider{client: client},
|
||||
log.New("storage.unified.migrator"),
|
||||
)
|
||||
}
|
||||
|
||||
func ProvideUnifiedMigratorParquet(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
client resourcepb.BulkStoreClient,
|
||||
) UnifiedMigrator {
|
||||
return newUnifiedMigrator(
|
||||
dashboardAccess,
|
||||
&bulkStoreClientStreamProvider{client: client},
|
||||
log.New("storage.unified.migrator.parquet"),
|
||||
)
|
||||
}
|
||||
|
||||
func newUnifiedMigrator(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
streamProvider streamProvider,
|
||||
log log.Logger,
|
||||
) UnifiedMigrator {
|
||||
return &unifiedMigration{
|
||||
MigrationDashboardAccessor: dashboardAccess,
|
||||
streamProvider: streamProvider,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// migrate function -- works for a single kind
|
||||
type migratorFunc = func(ctx context.Context, orgId int64, opts legacy.MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*legacy.BlobStoreInfo, error)
|
||||
|
||||
func (m *unifiedMigration) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
info, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts.Progress == nil {
|
||||
opts.Progress = func(count int, msg string) {} // noop
|
||||
}
|
||||
|
||||
// Validate counts for each resource type
|
||||
for _, summary := range response.Summary {
|
||||
legacyCount, err := m.getLegacyCount(sess, summary.Group, summary.Resource, summary.Namespace)
|
||||
if len(opts.Resources) < 1 {
|
||||
return nil, fmt.Errorf("missing resource selector")
|
||||
}
|
||||
|
||||
if opts.OnlyCount {
|
||||
return m.CountResources(ctx, opts)
|
||||
}
|
||||
|
||||
stream, err := m.streamProvider.createStream(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
migratorFuncs := []migratorFunc{}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateFolders)
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateLibraryPanels)
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateDashboards)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported resource: %s", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute migrations
|
||||
blobStore := legacy.BlobStoreInfo{}
|
||||
m.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
||||
for _, fn := range migratorFuncs {
|
||||
blobs, err := fn(ctx, info.OrgID, opts, stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get legacy count for %s/%s: %w", summary.Group, summary.Resource, err)
|
||||
m.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Account for rejected items in validation
|
||||
expectedCount := summary.Count + int64(len(response.Rejected))
|
||||
|
||||
m.log.Info("Count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace,
|
||||
"legacy_count", legacyCount,
|
||||
"unified_count", summary.Count,
|
||||
"rejected", len(response.Rejected),
|
||||
"history", summary.History)
|
||||
|
||||
// Validate that we migrated all items (allowing for rejected items)
|
||||
if legacyCount > expectedCount {
|
||||
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
|
||||
summary.Resource, summary.Group, summary.Namespace,
|
||||
legacyCount, summary.Count, len(response.Rejected))
|
||||
if blobs != nil {
|
||||
blobStore.Count += blobs.Count
|
||||
blobStore.Size += blobs.Size
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) getLegacyCount(sess *xorm.Session, group, resourceType, namespace string) (int64, error) {
|
||||
// Parse namespace to get org ID
|
||||
orgID, err := ParseOrgIDFromNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid namespace %s: %w", namespace, err)
|
||||
}
|
||||
|
||||
// Map group/resource to legacy table
|
||||
tableName, whereClause := m.getLegacyTableInfo(group, resourceType)
|
||||
if tableName == "" {
|
||||
return 0, fmt.Errorf("unknown resource type: %s.%s", resourceType, group)
|
||||
}
|
||||
|
||||
// Count items in legacy table using Table() before Count()
|
||||
count, err := sess.Table(tableName).Where(whereClause, orgID).Count()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count %s: %w", tableName, err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) getLegacyTableInfo(group, resource string) (table string, whereClause string) {
|
||||
// Map unified storage group/resource to legacy tables
|
||||
switch {
|
||||
case group == "dashboard.grafana.app" && resource == "dashboards":
|
||||
return "dashboard", "org_id = ? and is_folder = false"
|
||||
case group == "folder.grafana.app" && resource == "folders":
|
||||
return "dashboard", "org_id = ? and is_folder = true"
|
||||
case group == "playlist.grafana.app" && resource == "playlists":
|
||||
return "playlist", "org_id = ?"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
|
||||
// Use authlib to properly parse all namespace formats including "default" for org 1
|
||||
info, err := types.ParseNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse namespace: %w", err)
|
||||
}
|
||||
return info.OrgID, nil
|
||||
m.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
||||
return stream.CloseAndRecv()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
legacy "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
// MockUnifiedMigrator is an autogenerated mock type for the UnifiedMigrator type
|
||||
type MockUnifiedMigrator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockUnifiedMigrator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockUnifiedMigrator) EXPECT() *MockUnifiedMigrator_Expecter {
|
||||
return &MockUnifiedMigrator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Migrate provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockUnifiedMigrator) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Migrate")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, legacy.MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockUnifiedMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
|
||||
type MockUnifiedMigrator_Migrate_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Migrate is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts legacy.MigrateOptions
|
||||
func (_e *MockUnifiedMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockUnifiedMigrator_Migrate_Call {
|
||||
return &MockUnifiedMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) Run(run func(ctx context.Context, opts legacy.MigrateOptions)) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(legacy.MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) RunAndReturn(run func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockUnifiedMigrator creates a new instance of MockUnifiedMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockUnifiedMigrator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockUnifiedMigrator {
|
||||
mock := &MockUnifiedMigrator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// ValidationFunc is a function that validates migration results.
|
||||
// It receives the database session, migration response, and logger for reporting.
|
||||
// Return an error if validation fails, nil if validation passes or is skipped.
|
||||
type ValidationFunc func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error
|
||||
|
||||
// ResourceMigration handles migration of specific resource types from legacy to unified storage.
|
||||
// It implements migrator.CodeMigration and provides a generic, extensible way to migrate any
|
||||
// resource type by:
|
||||
//
|
||||
// 1. Iterating through all organizations
|
||||
// 2. For each org, delegating to LegacyMigrator to read from legacy and write to unified storage
|
||||
// 3. Validating migration results using the provided validation function (if any)
|
||||
//
|
||||
// To add a new resource type migration, simply create a new ResourceMigration instance in
|
||||
// service.go with the appropriate schema.GroupResource specifications and optional validation function.
|
||||
type ResourceMigration struct {
|
||||
migrator.MigrationBase
|
||||
migrator UnifiedMigrator
|
||||
resources []schema.GroupResource
|
||||
migrationID string
|
||||
validationFunc ValidationFunc // Optional: custom validation logic for this migration
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
// NewResourceMigration creates a new migration for the specified resources.
|
||||
// This is the primary way to register new resource migrations.
|
||||
//
|
||||
// Parameters:
|
||||
// - legacyMigrator: handles reading from legacy storage and writing to unified storage
|
||||
// - resources: list of GroupResource to migrate
|
||||
// - migrationID: unique identifier for this migration
|
||||
// - validationFunc: optional validation function to verify migration results.
|
||||
// If nil, no validation will be performed.
|
||||
//
|
||||
// Example with legacy table count validation:
|
||||
//
|
||||
// NewResourceMigration(
|
||||
// migrator,
|
||||
// []schema.GroupResource{{Group: "playlist.grafana.app", Resource: "playlists"}},
|
||||
// "playlists",
|
||||
// NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
// "playlist.grafana.app/playlists": {Table: "playlist", WhereClause: "org_id = ?"},
|
||||
// }),
|
||||
// )
|
||||
//
|
||||
// Example without validation:
|
||||
//
|
||||
// NewResourceMigration(migrator, resources, "new-resource", nil)
|
||||
func NewResourceMigration(
|
||||
migrator UnifiedMigrator,
|
||||
resources []schema.GroupResource,
|
||||
migrationID string,
|
||||
validationFunc ValidationFunc,
|
||||
) *ResourceMigration {
|
||||
return &ResourceMigration{
|
||||
migrator: migrator,
|
||||
resources: resources,
|
||||
migrationID: migrationID,
|
||||
validationFunc: validationFunc,
|
||||
log: log.New("storage.unified.resource_migration." + migrationID),
|
||||
}
|
||||
}
|
||||
|
||||
var _ migrator.CodeMigration = (*ResourceMigration)(nil)
|
||||
|
||||
// SQL implements migrator.Migration interface. Returns a description string.
|
||||
func (m *ResourceMigration) SQL(_ migrator.Dialect) string {
|
||||
return fmt.Sprintf("unified storage data migration: %s", m.migrationID)
|
||||
}
|
||||
|
||||
// Exec implements migrator.CodeMigration interface. Executes the migration across all organizations.
|
||||
func (m *ResourceMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
|
||||
ctx := context.Background()
|
||||
|
||||
orgs, err := m.getAllOrgs(sess)
|
||||
if err != nil {
|
||||
m.log.Error("failed to get organizations", "error", err)
|
||||
return fmt.Errorf("failed to get organizations: %w", err)
|
||||
}
|
||||
|
||||
if len(orgs) == 0 {
|
||||
m.log.Info("No organizations found to migrate, skipping migration")
|
||||
return nil
|
||||
}
|
||||
|
||||
m.log.Info("Starting migration for all organizations", "org_count", len(orgs), "resources", m.resources)
|
||||
|
||||
for _, org := range orgs {
|
||||
if err := m.migrateOrg(ctx, sess, org); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateOrg handles migration for a single organization
|
||||
func (m *ResourceMigration) migrateOrg(ctx context.Context, sess *xorm.Session, org orgInfo) error {
|
||||
namespace := types.OrgNamespaceFormatter(org.ID)
|
||||
m.log.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
|
||||
|
||||
// Create a service identity context for this namespace to authenticate with unified storage
|
||||
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Resources: m.resources,
|
||||
WithHistory: true, // Migrate with full history
|
||||
Progress: func(count int, msg string) {
|
||||
m.log.Info("Migration progress", "org_id", org.ID, "count", count, "message", msg)
|
||||
},
|
||||
}
|
||||
|
||||
// Execute the migration via legacy migrator
|
||||
response, err := m.migrator.Migrate(migrationCtx, opts)
|
||||
if err != nil {
|
||||
m.log.Error("Migration failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
|
||||
// Validate the migration results
|
||||
if err := m.validateMigration(sess, response); err != nil {
|
||||
m.log.Error("Migration validation failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration validation failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed for organization",
|
||||
"org_id", org.ID,
|
||||
"duration", time.Since(startTime),
|
||||
"processed", response.Processed,
|
||||
"summaries", len(response.Summary),
|
||||
"rejected", len(response.Rejected))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateMigration calls the custom validation function if provided
|
||||
func (m *ResourceMigration) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
|
||||
if m.validationFunc == nil {
|
||||
m.log.Debug("No validation function provided, skipping validation")
|
||||
return nil
|
||||
}
|
||||
|
||||
return m.validationFunc(sess, response, m.log)
|
||||
}
|
||||
|
||||
// LegacyTableInfo defines how to map a unified storage resource to its legacy table
|
||||
type LegacyTableInfo struct {
|
||||
Table string // Legacy table name (e.g., "dashboard", "playlist")
|
||||
WhereClause string // WHERE clause template with org_id parameter (e.g., "org_id = ? and is_folder = false")
|
||||
}
|
||||
|
||||
// NewLegacyTableCountValidator creates a ValidationFunc that validates migration by comparing
|
||||
// counts between legacy tables and unified storage.
|
||||
//
|
||||
// This is a helper for the common case of validating that all items from legacy tables
|
||||
// were successfully migrated to unified storage.
|
||||
//
|
||||
// Parameters:
|
||||
// - legacyTableMap: maps "group/resource" keys to LegacyTableInfo for validation.
|
||||
// Only resources with mappings will be validated.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// validator := NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
// "dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
|
||||
// "folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
|
||||
// })
|
||||
func NewLegacyTableCountValidator(legacyTableMap map[string]LegacyTableInfo) ValidationFunc {
|
||||
return func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error {
|
||||
// Check for rejected items
|
||||
if len(response.Rejected) > 0 {
|
||||
log.Warn("Migration had rejected items", "count", len(response.Rejected))
|
||||
for i, rejected := range response.Rejected {
|
||||
if i < 10 { // Log first 10 rejected items
|
||||
log.Warn("Rejected item",
|
||||
"namespace", rejected.Key.Namespace,
|
||||
"group", rejected.Key.Group,
|
||||
"resource", rejected.Key.Resource,
|
||||
"name", rejected.Key.Name,
|
||||
"reason", rejected.Error)
|
||||
}
|
||||
}
|
||||
// Rejections are not fatal - they may be expected for invalid data
|
||||
}
|
||||
|
||||
// Validate counts for each resource type
|
||||
for _, summary := range response.Summary {
|
||||
key := fmt.Sprintf("%s/%s", summary.Group, summary.Resource)
|
||||
tableInfo, ok := legacyTableMap[key]
|
||||
if !ok {
|
||||
log.Debug("No legacy table mapping for resource, skipping count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace)
|
||||
continue
|
||||
}
|
||||
|
||||
// Get legacy count
|
||||
orgID, err := ParseOrgIDFromNamespace(summary.Namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid namespace %s: %w", summary.Namespace, err)
|
||||
}
|
||||
|
||||
legacyCount, err := sess.Table(tableInfo.Table).Where(tableInfo.WhereClause, orgID).Count()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to count %s: %w", tableInfo.Table, err)
|
||||
}
|
||||
|
||||
// Account for rejected items in validation
|
||||
expectedCount := summary.Count + int64(len(response.Rejected))
|
||||
|
||||
log.Info("Count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace,
|
||||
"legacy_count", legacyCount,
|
||||
"unified_count", summary.Count,
|
||||
"rejected", len(response.Rejected),
|
||||
"history", summary.History)
|
||||
|
||||
// Validate that we migrated all items (allowing for rejected items)
|
||||
if legacyCount > expectedCount {
|
||||
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
|
||||
summary.Resource, summary.Group, summary.Namespace,
|
||||
legacyCount, summary.Count, len(response.Rejected))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
|
||||
// Use authlib to properly parse all namespace formats including "default" for org 1
|
||||
info, err := types.ParseNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse namespace: %w", err)
|
||||
}
|
||||
return info.OrgID, nil
|
||||
}
|
||||
|
||||
// orgInfo represents basic organization information
|
||||
type orgInfo struct {
|
||||
ID int64 `xorm:"id"`
|
||||
Name string `xorm:"name"`
|
||||
}
|
||||
|
||||
// getAllOrgs retrieves all organizations from the database
|
||||
func (m *ResourceMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
|
||||
var orgs []orgInfo
|
||||
err := sess.Table("org").Cols("id", "name").Find(&orgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orgs, nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/kvstore"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
|
||||
var logger = log.New("storage.unified.migrations")
|
||||
|
||||
type UnifiedStorageMigrationServiceImpl struct {
|
||||
migrator UnifiedMigrator
|
||||
cfg *setting.Cfg
|
||||
sqlStore db.DB
|
||||
kv kvstore.KVStore
|
||||
}
|
||||
|
||||
var _ contract.UnifiedStorageMigrationService = (*UnifiedStorageMigrationServiceImpl)(nil)
|
||||
|
||||
// ProvideUnifiedStorageMigrationService is a Wire provider that creates the migration service.
|
||||
// The service implements registry.BackgroundService and runs migrations during server startup.
|
||||
func ProvideUnifiedStorageMigrationService(
|
||||
migrator UnifiedMigrator,
|
||||
cfg *setting.Cfg,
|
||||
sqlStore db.DB,
|
||||
kv kvstore.KVStore,
|
||||
) contract.UnifiedStorageMigrationService {
|
||||
return &UnifiedStorageMigrationServiceImpl{
|
||||
migrator: migrator,
|
||||
cfg: cfg,
|
||||
sqlStore: sqlStore,
|
||||
kv: kv,
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes unified storage migrations as a background service.
|
||||
// This blocks until migrations complete. If migrations fail, an error is returned
|
||||
// which will prevent Grafana from starting.
|
||||
func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error {
|
||||
// TODO: temporary skip migrations in test environments to prevent integration test timeouts.
|
||||
if os.Getenv("GRAFANA_TEST_DB") != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// skip migrations if disabled in config
|
||||
if p.cfg.DisableDataMigrations {
|
||||
logger.Info("Data migrations are disabled, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: Re-enable once migrations are ready
|
||||
// TODO: add guarantee that this only runs once
|
||||
// return RegisterMigrations(p.migrator, p.cfg, p.sqlStore)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterMigrations initializes and registers all unified storage migrations.
|
||||
// This function is the entry point for all data migrations from legacy storage
|
||||
// to unified storage. It returns an error if migrations fail, preventing Grafana
|
||||
// from starting with inconsistent data.
|
||||
func RegisterMigrations(
|
||||
migrator UnifiedMigrator,
|
||||
cfg *setting.Cfg,
|
||||
sqlStore db.DB,
|
||||
) error {
|
||||
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
|
||||
defer span.End()
|
||||
mg := sqlstoremigrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unifiedstorage")
|
||||
mg.AddCreateMigration()
|
||||
|
||||
if err := prometheus.Register(mg); err != nil {
|
||||
logger.Warn("Failed to register migrator metrics", "error", err)
|
||||
}
|
||||
|
||||
// Register resource migrations
|
||||
// To add a new resource type, simply add another migration here with the appropriate resources
|
||||
registerResourceMigrations(mg, migrator)
|
||||
|
||||
// Run all registered migrations (blocking)
|
||||
sec := cfg.Raw.Section("database")
|
||||
if err := mg.RunMigrations(ctx,
|
||||
sec.Key("migration_locking").MustBool(true),
|
||||
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
|
||||
return fmt.Errorf("unified storage data migration failed: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("Unified storage migrations completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerResourceMigrations registers all unified storage resource migrations.
|
||||
// Add new resource types here by creating additional ResourceMigration instances.
|
||||
func registerResourceMigrations(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator) {
|
||||
dashboardsAndFolders := NewResourceMigration(
|
||||
migrator,
|
||||
[]schema.GroupResource{
|
||||
{Group: "folder.grafana.app", Resource: "folders"},
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
},
|
||||
"folders-dashboards",
|
||||
NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
"folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
|
||||
"dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
|
||||
}),
|
||||
)
|
||||
mg.AddMigration("folders and dashboards migration", dashboardsAndFolders)
|
||||
}
|
||||
Reference in New Issue
Block a user