diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go index 8da13ca5341..b64111ebffc 100644 --- a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go +++ b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go @@ -9,6 +9,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/urfave/cli/v2" "k8s.io/apimachinery/pkg/runtime/schema" authlib "github.com/grafana/authlib/types" @@ -16,6 +17,7 @@ import ( dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/tracing" @@ -52,8 +54,10 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err }, LargeObjects: nil, // TODO... from config Progress: func(count int, msg string) { - if count < 1 || time.Since(last) > time.Second { - fmt.Printf("[%4d] %s\n", count, msg) + const minInterval = time.Second + shouldPrint := count < 1 || time.Since(last) > minInterval + if shouldPrint { + logger.Info(fmt.Sprintf("[%4d] %s", count, msg)) last = time.Now() } }, @@ -67,7 +71,10 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err migrator := legacy.NewDashboardAccess( legacysql.NewDatabaseProvider(sqlStore), authlib.OrgNamespaceFormatter, - nil, provisioning, nil, sort.ProvideService(), + nil, // no dashboards.Store + provisioning, + nil, // no librarypanels.Service + sort.ProvideService(), ) if c.Bool("non-interactive") { @@ -80,12 +87,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err opts.BlobStore = client rsp, err := migrator.Migrate(ctx, opts) if err != nil { - return err + msg := fmt.Sprintf("Failed to migrate legacy resources: %+v", err) + return cli.Exit(msg, 1) } - fmt.Printf("Unified storage export: %s\n", time.Since(start)) + logger.Info("Migrated legacy resources successfully in", time.Since(start)) if rsp != nil { jj, _ := json.MarshalIndent(rsp, "", " ") - fmt.Printf("%s\n", string(jj)) + logger.Info("Migration summary:", string(jj)) } return nil } diff --git a/pkg/registry/apis/dashboard/legacy/migrate.go b/pkg/registry/apis/dashboard/legacy/migrate.go index 6fd0225f45e..8cbaeafaf0e 100644 --- a/pkg/registry/apis/dashboard/legacy/migrate.go +++ b/pkg/registry/apis/dashboard/legacy/migrate.go @@ -59,7 +59,7 @@ type BlobStoreInfo struct { } // migrate function -- works for a single kind -type migrator = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) +type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { info, err := authlib.ParseNamespace(opts.Namespace) @@ -75,7 +75,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( return nil, fmt.Errorf("missing resource selector") } - migrators := []migrator{} + migratorFuncs := []migratorFunc{} settings := resource.BulkSettings{ RebuildCollection: true, SkipValidation: true, @@ -84,7 +84,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( for _, res := range opts.Resources { switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { case "folder.grafana.app/folders": - migrators = append(migrators, a.migrateFolders) + migratorFuncs = append(migratorFuncs, a.migrateFolders) settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ Namespace: opts.Namespace, Group: folders.GROUP, @@ -92,7 +92,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( }) case "dashboard.grafana.app/librarypanels": - migrators = append(migrators, a.migratePanels) + migratorFuncs = append(migratorFuncs, a.migratePanels) settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ Namespace: opts.Namespace, Group: dashboard.GROUP, @@ -100,7 +100,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( }) case "dashboard.grafana.app/dashboards": - migrators = append(migrators, a.migrateDashboards) + migratorFuncs = append(migratorFuncs, a.migrateDashboards) settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ Namespace: opts.Namespace, Group: dashboard.GROUP, @@ -110,12 +110,22 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( return nil, fmt.Errorf("unsupported resource: %s", res) } } - if opts.OnlyCount { return a.countValues(ctx, opts) } ctx = metadata.NewOutgoingContext(ctx, settings.ToMD()) + if md, ok := metadata.FromOutgoingContext(ctx); ok { + a.log.Debug("bulk grpc request metadata", + "metadata", md, + "collection", settings.Collection, + ) + } else { + a.log.Debug("bulk grpc request, no metadata found", + "collection", settings.Collection, + ) + } + stream, err := opts.Store.BulkProcess(ctx) if err != nil { return nil, err @@ -123,9 +133,11 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( // Now run each migration blobStore := BlobStoreInfo{} - for _, m := range migrators { + a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID) + for _, m := range migratorFuncs { blobs, err := m(ctx, info.OrgID, opts, stream) if err != nil { + a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace) return nil, err } if blobs != nil { @@ -133,7 +145,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) ( blobStore.Size += blobs.Size } } - fmt.Printf("BLOBS: %+v\n", blobStore) + a.log.Info("finished migrating legacy resources", "blobStore", blobStore) return stream.CloseAndRecv() } @@ -291,7 +303,11 @@ func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, if len(rows.rejected) > 0 { for _, row := range rows.rejected { id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID] - fmt.Printf("REJECTED: %s / %s\n", id, row.Dash.Name) + a.log.Warn("rejected dashboard", + "dashboard", row.Dash.Name, + "uid", row.Dash.UID, + "id", id, + ) opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name)) } }