public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: add logging around migration cli (#107046)

Browse Source
This commit is contained in:
Mustafa Sencer Özcan
2025-06-20 15:20:06 +00:00
committed by GitHub
parent 5d3a1f07c8
commit 3a13c2db64
2 changed files with 39 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go
+14 -6
View File
@@ -9,6 +9,7 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/urfave/cli/v2"
"k8s.io/apimachinery/pkg/runtime/schema"
authlib "github.com/grafana/authlib/types"
@@ -16,6 +17,7 @@ import (
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -52,8 +54,10 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
},
LargeObjects: nil, // TODO... from config
Progress: func(count int, msg string) {
if count < 1 || time.Since(last) > time.Second {
fmt.Printf("[%4d] %s\n", count, msg)
const minInterval = time.Second
shouldPrint := count < 1 || time.Since(last) > minInterval
if shouldPrint {
logger.Info(fmt.Sprintf("[%4d] %s", count, msg))
last = time.Now()
}
},
@@ -67,7 +71,10 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
migrator := legacy.NewDashboardAccess(
legacysql.NewDatabaseProvider(sqlStore),
authlib.OrgNamespaceFormatter,
nil, provisioning, nil, sort.ProvideService(),
nil, // no dashboards.Store
provisioning,
nil, // no librarypanels.Service
sort.ProvideService(),
)
if c.Bool("non-interactive") {
@@ -80,12 +87,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
opts.BlobStore = client
rsp, err := migrator.Migrate(ctx, opts)
if err != nil {
return err
msg := fmt.Sprintf("Failed to migrate legacy resources: %+v", err)
return cli.Exit(msg, 1)
}
fmt.Printf("Unified storage export: %s\n", time.Since(start))
logger.Info("Migrated legacy resources successfully in", time.Since(start))
if rsp != nil {
jj, _ := json.MarshalIndent(rsp, "", " ")
fmt.Printf("%s\n", string(jj))
logger.Info("Migration summary:", string(jj))
}
return nil
}
pkg/registry/apis/dashboard/legacy/migrate.go
+25 -9
View File
@@ -59,7 +59,7 @@ type BlobStoreInfo struct {
}
// migrate function -- works for a single kind
type migrator = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
info, err := authlib.ParseNamespace(opts.Namespace)
@@ -75,7 +75,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
return nil, fmt.Errorf("missing resource selector")
}
migrators := []migrator{}
migratorFuncs := []migratorFunc{}
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
@@ -84,7 +84,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
migrators = append(migrators, a.migrateFolders)
migratorFuncs = append(migratorFuncs, a.migrateFolders)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
@@ -92,7 +92,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
})
case "dashboard.grafana.app/librarypanels":
migrators = append(migrators, a.migratePanels)
migratorFuncs = append(migratorFuncs, a.migratePanels)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
@@ -100,7 +100,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
})
case "dashboard.grafana.app/dashboards":
migrators = append(migrators, a.migrateDashboards)
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
@@ -110,12 +110,22 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
return nil, fmt.Errorf("unsupported resource: %s", res)
}
}
if opts.OnlyCount {
return a.countValues(ctx, opts)
}
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
if md, ok := metadata.FromOutgoingContext(ctx); ok {
a.log.Debug("bulk grpc request metadata",
"metadata", md,
"collection", settings.Collection,
)
} else {
a.log.Debug("bulk grpc request, no metadata found",
"collection", settings.Collection,
)
}
stream, err := opts.Store.BulkProcess(ctx)
if err != nil {
return nil, err
@@ -123,9 +133,11 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
// Now run each migration
blobStore := BlobStoreInfo{}
for _, m := range migrators {
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
for _, m := range migratorFuncs {
blobs, err := m(ctx, info.OrgID, opts, stream)
if err != nil {
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
return nil, err
}
if blobs != nil {
@@ -133,7 +145,7 @@ func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (
blobStore.Size += blobs.Size
}
}
fmt.Printf("BLOBS: %+v\n", blobStore)
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
return stream.CloseAndRecv()
}
@@ -291,7 +303,11 @@ func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64,
if len(rows.rejected) > 0 {
for _, row := range rows.rejected {
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
fmt.Printf("REJECTED: %s / %s\n", id, row.Dash.Name)
a.log.Warn("rejected dashboard",
"dashboard", row.Dash.Name,
"uid", row.Dash.UID,
"id", id,
)
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
}
}