feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
co-authored by
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources/signature"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/parquet"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
@@ -29,7 +30,7 @@ type LegacyResourcesMigrator interface {
|
||||
type legacyResourcesMigrator struct {
|
||||
repositoryResources resources.RepositoryResourcesFactory
|
||||
parsers resources.ParserFactory
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
signerFactory signature.SignerFactory
|
||||
clients resources.ClientFactory
|
||||
exportFn export.ExportFn
|
||||
@@ -38,7 +39,7 @@ type legacyResourcesMigrator struct {
|
||||
func NewLegacyResourcesMigrator(
|
||||
repositoryResources resources.RepositoryResourcesFactory,
|
||||
parsers resources.ParserFactory,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
signerFactory signature.SignerFactory,
|
||||
clients resources.ClientFactory,
|
||||
exportFn export.ExportFn,
|
||||
@@ -46,7 +47,7 @@ func NewLegacyResourcesMigrator(
|
||||
return &legacyResourcesMigrator{
|
||||
repositoryResources: repositoryResources,
|
||||
parsers: parsers,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
signerFactory: signerFactory,
|
||||
clients: clients,
|
||||
exportFn: exportFn,
|
||||
@@ -94,7 +95,7 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
|
||||
reader := newLegacyResourceMigrator(
|
||||
rw,
|
||||
m.legacyMigrator,
|
||||
m.dashboardAccess,
|
||||
parser,
|
||||
repositoryResources,
|
||||
progress,
|
||||
@@ -113,21 +114,21 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
}
|
||||
|
||||
type legacyResourceResourceMigrator struct {
|
||||
repo repository.ReaderWriter
|
||||
legacy legacy.LegacyMigrator
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
repo repository.ReaderWriter
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
}
|
||||
|
||||
func newLegacyResourceMigrator(
|
||||
repo repository.ReaderWriter,
|
||||
legacy legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
parser resources.Parser,
|
||||
resources resources.RepositoryResources,
|
||||
progress jobs.JobProgressRecorder,
|
||||
@@ -141,16 +142,16 @@ func newLegacyResourceMigrator(
|
||||
history = make(map[string]string)
|
||||
}
|
||||
return &legacyResourceResourceMigrator{
|
||||
repo: repo,
|
||||
legacy: legacy,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
repo: repo,
|
||||
dashboardAccess: dashboardAccess,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,14 +226,21 @@ func (r *legacyResourceResourceMigrator) Write(ctx context.Context, key *resourc
|
||||
|
||||
func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
r.progress.SetMessage(ctx, fmt.Sprintf("migrate %s resource", r.kind.Resource))
|
||||
|
||||
// Create a parquet migrator with this instance as the BulkResourceWriter
|
||||
parquetClient := parquet.NewBulkResourceWriterClient(r)
|
||||
migrator := unifiedmigrations.ProvideUnifiedMigratorParquet(
|
||||
r.dashboardAccess,
|
||||
parquetClient,
|
||||
)
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: r.namespace,
|
||||
WithHistory: r.options.History,
|
||||
Resources: []schema.GroupResource{r.kind},
|
||||
Store: parquet.NewBulkResourceWriterClient(r),
|
||||
OnlyCount: true, // first get the count
|
||||
}
|
||||
stats, err := r.legacy.Migrate(ctx, opts)
|
||||
stats, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to count legacy items %w", err)
|
||||
}
|
||||
@@ -248,7 +256,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
opts.OnlyCount = false // this time actually write
|
||||
_, err = r.legacy.Migrate(ctx, opts)
|
||||
_, err = migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate legacy %s: %w", r.kind.Resource, err)
|
||||
}
|
||||
|
||||
@@ -93,8 +93,8 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
mockRepoResourcesFactory.On("Client", mock.Anything, mock.Anything).
|
||||
Return(mockRepoResources, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("legacy migrator error"))
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
signerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -136,7 +136,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
@@ -308,22 +308,18 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
History: true,
|
||||
}).Return(mockSigner, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
// Mock CountResources for the count phase
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once()
|
||||
// Mock MigrateDashboards for the actual migration phase (dashboards resource)
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Count: 10,
|
||||
History: 5,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Migration phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{
|
||||
Count: 10,
|
||||
Size: 5,
|
||||
}, nil).Once()
|
||||
|
||||
mockClients := resources.NewMockResourceClients(t)
|
||||
mockClientFactory := resources.NewMockClientFactory(t)
|
||||
@@ -339,7 +335,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
mockSignerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -362,7 +358,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
@@ -742,8 +738,8 @@ func TestLegacyResourceResourceMigrator_Write(t *testing.T) {
|
||||
|
||||
func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
t.Run("should fail when legacy migrate count fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("count error"))
|
||||
|
||||
@@ -752,7 +748,7 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
@@ -766,89 +762,91 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to count legacy items")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should fail when legacy migrate write fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
// For test-resources GroupResource, we don't know which method it will call, but since it's not dashboards/folders/librarypanels,
|
||||
// the Migrate will fail trying to map the resource type. Let's make it dashboards for this test.
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("write error")).Once() // Write phase
|
||||
}), mock.Anything).Return(nil, errors.New("write error")).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "test-resources"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "migrate legacy test-resources: write error")
|
||||
require.Contains(t, err.Error(), "migrate legacy dashboards: write error")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should successfully migrate resource", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to history if history is greater than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 1,
|
||||
History: 100,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -856,39 +854,39 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to count if history is less than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 200,
|
||||
History: 1,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -897,20 +895,20 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signer,
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
@@ -109,7 +110,7 @@ type APIBuilder struct {
|
||||
jobHistoryConfig *JobHistoryConfig
|
||||
jobHistoryLoki *jobs.LokiJobHistory
|
||||
resourceLister resources.ResourceLister
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
storageStatus dualwrite.Service
|
||||
unified resource.ResourceClient
|
||||
repoFactory repository.Factory
|
||||
@@ -135,7 +136,7 @@ func NewAPIBuilder(
|
||||
features featuremgmt.FeatureToggles,
|
||||
unified resource.ResourceClient,
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
access authlib.AccessChecker,
|
||||
@@ -158,6 +159,7 @@ func NewAPIBuilder(
|
||||
clients = resources.NewClientFactory(configProvider)
|
||||
}
|
||||
parsers := resources.NewParserFactory(clients)
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(dashboardAccess, unified)
|
||||
resourceLister := resources.NewResourceListerForMigrations(unified, legacyMigrator, storageStatus)
|
||||
|
||||
b := &APIBuilder{
|
||||
@@ -170,7 +172,7 @@ func NewAPIBuilder(
|
||||
parsers: parsers,
|
||||
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
|
||||
resourceLister: resourceLister,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
storageStatus: storageStatus,
|
||||
unified: unified,
|
||||
access: access,
|
||||
@@ -234,7 +236,7 @@ func RegisterAPIService(
|
||||
client resource.ResourceClient, // implements resource.RepositoryClient
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
access authlib.AccessClient,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
tracer tracing.Tracer,
|
||||
@@ -258,7 +260,7 @@ func RegisterAPIService(
|
||||
features,
|
||||
client,
|
||||
configProvider,
|
||||
legacyMigrator, storageStatus,
|
||||
dashboardAccess, storageStatus,
|
||||
usageStats,
|
||||
access,
|
||||
tracer,
|
||||
@@ -722,7 +724,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
|
||||
legacyResources := migrate.NewLegacyResourcesMigrator(
|
||||
b.repositoryResources,
|
||||
b.parsers,
|
||||
b.legacyMigrator,
|
||||
b.dashboardAccess,
|
||||
signerFactory,
|
||||
b.clients,
|
||||
export.ExportAll,
|
||||
@@ -1241,8 +1243,9 @@ func (b *APIBuilder) tryRunningOnlyUnifiedStorage() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Count how many things exist
|
||||
rsp, err := b.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
// Count how many things exist - create a migrator on-demand for this
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(b.dashboardAccess, b.unified)
|
||||
rsp, err := legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: "default", // FIXME! this works for single org, but need to check multi-org
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
@@ -30,9 +31,9 @@ type ResourceStore interface {
|
||||
}
|
||||
|
||||
type ResourceListerFromSearch struct {
|
||||
store ResourceStore
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
storageStatus dualwrite.Service
|
||||
store ResourceStore
|
||||
migrator migrations.UnifiedMigrator
|
||||
storageStatus dualwrite.Service
|
||||
}
|
||||
|
||||
func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
@@ -42,13 +43,13 @@ func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
// FIXME: the logic about migration and storage should probably be separated from this
|
||||
func NewResourceListerForMigrations(
|
||||
store ResourceStore,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
migrator migrations.UnifiedMigrator,
|
||||
storageStatus dualwrite.Service,
|
||||
) ResourceLister {
|
||||
return &ResourceListerFromSearch{
|
||||
store: store,
|
||||
legacyMigrator: legacyMigrator,
|
||||
storageStatus: storageStatus,
|
||||
store: store,
|
||||
migrator: migrator,
|
||||
storageStatus: storageStatus,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,8 +134,8 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit
|
||||
}
|
||||
|
||||
// Get the stats based on what a migration could support
|
||||
if o.storageStatus != nil && o.legacyMigrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
if o.storageStatus != nil && o.migrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.migrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
Reference in New Issue
Block a user