diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/stubs.go b/pkg/cmd/grafana-cli/commands/datamigrations/stubs.go deleted file mode 100644 index 22dc15d6fc7..00000000000 --- a/pkg/cmd/grafana-cli/commands/datamigrations/stubs.go +++ /dev/null @@ -1,70 +0,0 @@ -package datamigrations - -import ( - "context" - "path/filepath" - - "github.com/grafana/grafana/pkg/services/provisioning" - "github.com/grafana/grafana/pkg/services/provisioning/dashboards" -) - -var ( - _ provisioning.ProvisioningService = (*stubProvisioning)(nil) -) - -func newStubProvisioning(path string) (provisioning.ProvisioningService, error) { - cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards")) - if err != nil { - return nil, err - } - stub := &stubProvisioning{ - path: make(map[string]string), - } - for _, cfg := range cfgs { - stub.path[cfg.Name] = cfg.Options["path"].(string) - } - return &stubProvisioning{}, nil -} - -type stubProvisioning struct { - path map[string]string // name > options.path -} - -// GetAllowUIUpdatesFromConfig implements provisioning.ProvisioningService. -func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool { - return false -} - -func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string { - return s.path[name] -} - -// ProvisionAlerting implements provisioning.ProvisioningService. -func (s *stubProvisioning) ProvisionAlerting(ctx context.Context) error { - panic("unimplemented") -} - -// ProvisionDashboards implements provisioning.ProvisioningService. -func (s *stubProvisioning) ProvisionDashboards(ctx context.Context) error { - panic("unimplemented") -} - -// ProvisionDatasources implements provisioning.ProvisioningService. -func (s *stubProvisioning) ProvisionDatasources(ctx context.Context) error { - panic("unimplemented") -} - -// ProvisionPlugins implements provisioning.ProvisioningService. -func (s *stubProvisioning) ProvisionPlugins(ctx context.Context) error { - panic("unimplemented") -} - -// Run implements provisioning.ProvisioningService. -func (s *stubProvisioning) Run(ctx context.Context) error { - panic("unimplemented") -} - -// RunInitProvisioners implements provisioning.ProvisioningService. -func (s *stubProvisioning) RunInitProvisioners(ctx context.Context) error { - panic("unimplemented") -} diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go index f6bdc211f15..2e8a9892ab1 100644 --- a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go +++ b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go @@ -24,10 +24,11 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/search/sort" + "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/legacysql" "github.com/grafana/grafana/pkg/storage/unified" + "github.com/grafana/grafana/pkg/storage/unified/migrations" "github.com/grafana/grafana/pkg/storage/unified/parquet" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" @@ -52,7 +53,6 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err {Group: folders.GROUP, Resource: folders.RESOURCE}, {Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE}, }, - LargeObjects: nil, // TODO... from config Progress: func(count int, msg string) { const minInterval = time.Second shouldPrint := count < 1 || time.Since(last) > minInterval @@ -69,31 +69,26 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err } featureToggles := featuremgmt.ProvideToggles(featureManager) - provisioning, err := newStubProvisioning(cfg.ProvisioningPath) + provisioning, err := provisioning.ProvideStubProvisioningService(cfg) if err != nil { return err } - migrator := legacy.NewDashboardAccess( + grpcClient, err := newUnifiedClient(cfg, sqlStore, featureToggles) + if err != nil { + return err + } + + dashboardAccess := legacy.ProvideMigratorDashboardAccessor( legacysql.NewDatabaseProvider(sqlStore), - authlib.OrgNamespaceFormatter, - nil, // no dashboards.Store provisioning, - nil, // no librarypanels.Service - sort.ProvideService(), - nil, // we don't delete during migration, and this is only need to delete permission. acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), featureToggles, ) - client, err := newUnifiedClient(cfg, sqlStore, featureToggles) - if err != nil { - return err - } - if c.Bool("non-interactive") { - opts.Store = client - opts.BlobStore = client + migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient) + opts.WithHistory = true // always include history in non-interactive mode rsp, err := migrator.Migrate(ctx, opts) if exitErr := handleMigrationError(err, rsp); exitErr != nil { @@ -113,6 +108,8 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err return err } if yes { + migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, nil) // no need for grpc client for counting + opts.OnlyCount = true rsp, err := migrator.Migrate(ctx, opts) if err != nil { @@ -141,12 +138,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err if err != nil { return err } - start = time.Now() - last = time.Now() - opts.Store, err = newParquetClient(file) + parquetClient, err := newParquetClient(file) if err != nil { return err } + migrator := migrations.ProvideUnifiedMigratorParquet(dashboardAccess, parquetClient) + start = time.Now() + last = time.Now() rsp, err := migrator.Migrate(ctx, opts) if err != nil { return err @@ -172,7 +170,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err req.Kinds = append(req.Kinds, fmt.Sprintf("%s/%s", r.Group, r.Resource)) } - stats, err := client.GetStats(ctx, req) + stats, err := grpcClient.GetStats(ctx, req) if err != nil { return err } @@ -188,10 +186,9 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err return err } if yes { + migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient) start = time.Now() last = time.Now() - opts.Store = client - opts.BlobStore = client rsp, err := migrator.Migrate(ctx, opts) if err != nil { return err diff --git a/pkg/registry/apis/dashboard/legacy/legacy_migrator_mock.go b/pkg/registry/apis/dashboard/legacy/legacy_migrator_mock.go deleted file mode 100644 index bb7231e86f6..00000000000 --- a/pkg/registry/apis/dashboard/legacy/legacy_migrator_mock.go +++ /dev/null @@ -1,96 +0,0 @@ -// Code generated by mockery v2.53.4. DO NOT EDIT. - -package legacy - -import ( - context "context" - - resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb" - mock "github.com/stretchr/testify/mock" -) - -// MockLegacyMigrator is an autogenerated mock type for the LegacyMigrator type -type MockLegacyMigrator struct { - mock.Mock -} - -type MockLegacyMigrator_Expecter struct { - mock *mock.Mock -} - -func (_m *MockLegacyMigrator) EXPECT() *MockLegacyMigrator_Expecter { - return &MockLegacyMigrator_Expecter{mock: &_m.Mock} -} - -// Migrate provides a mock function with given fields: ctx, opts -func (_m *MockLegacyMigrator) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { - ret := _m.Called(ctx, opts) - - if len(ret) == 0 { - panic("no return value specified for Migrate") - } - - var r0 *resourcepb.BulkResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok { - return rf(ctx, opts) - } - if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok { - r0 = rf(ctx, opts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*resourcepb.BulkResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok { - r1 = rf(ctx, opts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MockLegacyMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate' -type MockLegacyMigrator_Migrate_Call struct { - *mock.Call -} - -// Migrate is a helper method to define mock.On call -// - ctx context.Context -// - opts MigrateOptions -func (_e *MockLegacyMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockLegacyMigrator_Migrate_Call { - return &MockLegacyMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)} -} - -func (_c *MockLegacyMigrator_Migrate_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockLegacyMigrator_Migrate_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(MigrateOptions)) - }) - return _c -} - -func (_c *MockLegacyMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockLegacyMigrator_Migrate_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *MockLegacyMigrator_Migrate_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockLegacyMigrator_Migrate_Call { - _c.Call.Return(run) - return _c -} - -// NewMockLegacyMigrator creates a new instance of MockLegacyMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewMockLegacyMigrator(t interface { - mock.TestingT - Cleanup(func()) -}) *MockLegacyMigrator { - mock := &MockLegacyMigrator{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/pkg/registry/apis/dashboard/legacy/migrate.go b/pkg/registry/apis/dashboard/legacy/migrate.go deleted file mode 100644 index e43a0fb7629..00000000000 --- a/pkg/registry/apis/dashboard/legacy/migrate.go +++ /dev/null @@ -1,460 +0,0 @@ -package legacy - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - - "google.golang.org/grpc/metadata" - "k8s.io/apimachinery/pkg/runtime/schema" - - authlib "github.com/grafana/authlib/types" - - dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" - folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" - "github.com/grafana/grafana/pkg/apimachinery/utils" - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/librarypanels" - "github.com/grafana/grafana/pkg/services/provisioning" - "github.com/grafana/grafana/pkg/services/search/sort" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/storage/legacysql" - "github.com/grafana/grafana/pkg/storage/unified/apistore" - "github.com/grafana/grafana/pkg/storage/unified/resource" - "github.com/grafana/grafana/pkg/storage/unified/resourcepb" -) - -type MigrateOptions struct { - Namespace string - Store resourcepb.BulkStoreClient - LargeObjects apistore.LargeObjectSupport - BlobStore resourcepb.BlobStoreClient - Resources []schema.GroupResource - WithHistory bool // only applies to dashboards - OnlyCount bool // just count the values - Progress func(count int, msg string) -} - -// Read from legacy and write into unified storage -// -//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter -type LegacyMigrator interface { - Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) -} - -// This can migrate Folders, Dashboards and LibraryPanels -func ProvideLegacyMigrator( - sql db.DB, // direct access to tables - provisioning provisioning.ProvisioningService, // only needed for dashboard settings - libraryPanelSvc librarypanels.Service, - dashboardPermissionSvc accesscontrol.DashboardPermissionsService, - accessControl accesscontrol.AccessControl, - features featuremgmt.FeatureToggles, -) LegacyMigrator { - dbp := legacysql.NewDatabaseProvider(sql) - return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), dashboardPermissionSvc, accessControl, features) -} - -type BlobStoreInfo struct { - Count int64 - Size int64 -} - -// migrate function -- works for a single kind -type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) - -func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { - info, err := authlib.ParseNamespace(opts.Namespace) - if err != nil { - return nil, err - } - if opts.Progress == nil { - opts.Progress = func(count int, msg string) {} // noop - } - - // Migrate everything - if len(opts.Resources) < 1 { - return nil, fmt.Errorf("missing resource selector") - } - - migratorFuncs := []migratorFunc{} - settings := resource.BulkSettings{ - RebuildCollection: true, - SkipValidation: true, - } - - for _, res := range opts.Resources { - switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { - case "folder.grafana.app/folders": - migratorFuncs = append(migratorFuncs, a.migrateFolders) - settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: folders.GROUP, - Resource: folders.RESOURCE, - }) - - case "dashboard.grafana.app/librarypanels": - migratorFuncs = append(migratorFuncs, a.migratePanels) - settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: dashboard.GROUP, - Resource: dashboard.LIBRARY_PANEL_RESOURCE, - }) - - case "dashboard.grafana.app/dashboards": - migratorFuncs = append(migratorFuncs, a.migrateDashboards) - settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: dashboard.GROUP, - Resource: dashboard.DASHBOARD_RESOURCE, - }) - default: - return nil, fmt.Errorf("unsupported resource: %s", res) - } - } - if opts.OnlyCount { - return a.countValues(ctx, opts) - } - - ctx = metadata.NewOutgoingContext(ctx, settings.ToMD()) - if md, ok := metadata.FromOutgoingContext(ctx); ok { - a.log.Debug("bulk grpc request metadata", - "metadata", md, - "collection", settings.Collection, - ) - } else { - a.log.Debug("bulk grpc request, no metadata found", - "collection", settings.Collection, - ) - } - - stream, err := opts.Store.BulkProcess(ctx) - if err != nil { - return nil, err - } - - // Now run each migration - blobStore := BlobStoreInfo{} - a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID) - for _, m := range migratorFuncs { - blobs, err := m(ctx, info.OrgID, opts, stream) - if err != nil { - a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace) - return nil, err - } - if blobs != nil { - blobStore.Count += blobs.Count - blobStore.Size += blobs.Size - } - } - a.log.Info("finished migrating legacy resources", "blobStore", blobStore) - return stream.CloseAndRecv() -} - -func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { - sql, err := a.sql(ctx) - if err != nil { - return nil, err - } - ns, err := authlib.ParseNamespace(opts.Namespace) - if err != nil { - return nil, err - } - orgId := ns.OrgID - rsp := &resourcepb.BulkResponse{} - err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - for _, res := range opts.Resources { - switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { - case "folder.grafana.app/folders": - summary := &resourcepb.BulkResponse_Summary{} - summary.Group = folders.GROUP - summary.Group = folders.RESOURCE - _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+ - " WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count) - rsp.Summary = append(rsp.Summary, summary) - - case "dashboard.grafana.app/librarypanels": - summary := &resourcepb.BulkResponse_Summary{} - summary.Group = dashboard.GROUP - summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE - _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+ - " WHERE org_id=?", orgId).Get(&summary.Count) - rsp.Summary = append(rsp.Summary, summary) - - case "dashboard.grafana.app/dashboards": - summary := &resourcepb.BulkResponse_Summary{} - summary.Group = dashboard.GROUP - summary.Resource = dashboard.DASHBOARD_RESOURCE - rsp.Summary = append(rsp.Summary, summary) - - _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+ - " WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count) - if err != nil { - return err - } - - // Also count history - _, err = sess.SQL(`SELECT COUNT(*) - FROM `+sql.Table("dashboard_version")+` as dv - JOIN `+sql.Table("dashboard")+` as dd - ON dd.id = dv.dashboard_id - WHERE org_id=?`, orgId).Get(&summary.History) - } - if err != nil { - return err - } - } - return nil - }) - return rsp, nil -} - -func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { - query := &DashboardQuery{ - OrgID: orgId, - Limit: 100000000, - GetHistory: opts.WithHistory, // include history - AllowFallback: true, // allow fallback to dashboard table during migration - Order: "ASC", // oldest first - } - - blobs := &BlobStoreInfo{} - sql, err := a.sql(ctx) - if err != nil { - return blobs, err - } - - opts.Progress(-1, "migrating dashboards...") - rows, err := a.getRows(ctx, sql, query) - if rows != nil { - defer func() { - _ = rows.Close() - }() - } - if err != nil { - return blobs, err - } - - large := opts.LargeObjects - - // Now send each dashboard - for i := 1; rows.Next(); i++ { - dash := rows.row.Dash - if dash.APIVersion == "" { - dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP) - } - dash.SetNamespace(opts.Namespace) - dash.SetResourceVersion("") // it will be filled in by the backend - - body, err := json.Marshal(dash) - if err != nil { - err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err) - return blobs, err - } - - req := &resourcepb.BulkRequest{ - Key: &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: dashboard.GROUP, - Resource: dashboard.DASHBOARD_RESOURCE, - Name: rows.Name(), - }, - Value: body, - Folder: rows.row.FolderUID, - Action: resourcepb.BulkRequest_ADDED, - } - if dash.Generation > 1 { - req.Action = resourcepb.BulkRequest_MODIFIED - } else if dash.Generation < 0 { - req.Action = resourcepb.BulkRequest_DELETED - } - - // With large object support - if large != nil && len(body) > large.Threshold() { - obj, err := utils.MetaAccessor(dash) - if err != nil { - return blobs, err - } - - opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body))) - err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value) - if err != nil { - return blobs, err - } - - // The smaller version (most of spec removed) - req.Value, err = json.Marshal(dash) - if err != nil { - return blobs, err - } - blobs.Count++ - blobs.Size += int64(len(body)) - } - - opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count)) - - err = stream.Send(req) - if err != nil { - if errors.Is(err, io.EOF) { - opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i)) - err = nil - } - return blobs, err - } - } - - if len(rows.rejected) > 0 { - for _, row := range rows.rejected { - id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID] - a.log.Warn("rejected dashboard", - "namespace", opts.Namespace, - "dashboard", row.Dash.Name, - "uid", row.Dash.UID, - "id", id, - "version", row.Dash.Generation, - ) - opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name)) - } - } - - if rows.Error() != nil { - return blobs, rows.Error() - } - - opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count)) - return blobs, err -} - -func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { - query := &DashboardQuery{ - OrgID: orgId, - Limit: 100000000, - GetFolders: true, - Order: "ASC", - } - - sql, err := a.sql(ctx) - if err != nil { - return nil, err - } - - opts.Progress(-1, "migrating folders...") - rows, err := a.getRows(ctx, sql, query) - if rows != nil { - defer func() { - _ = rows.Close() - }() - } - if err != nil { - return nil, err - } - - // Now send each dashboard - for i := 1; rows.Next(); i++ { - dash := rows.row.Dash - dash.APIVersion = "folder.grafana.app/v1beta1" - dash.Kind = "Folder" - dash.SetNamespace(opts.Namespace) - dash.SetResourceVersion("") // it will be filled in by the backend - - spec := map[string]any{ - "title": dash.Spec.Object["title"], - } - description := dash.Spec.Object["description"] - if description != nil { - spec["description"] = description - } - dash.Spec.Object = spec - - body, err := json.Marshal(dash) - if err != nil { - return nil, err - } - - req := &resourcepb.BulkRequest{ - Key: &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: "folder.grafana.app", - Resource: "folders", - Name: rows.Name(), - }, - Value: body, - Folder: rows.row.FolderUID, - Action: resourcepb.BulkRequest_ADDED, - } - if dash.Generation > 1 { - req.Action = resourcepb.BulkRequest_MODIFIED - } else if dash.Generation < 0 { - req.Action = resourcepb.BulkRequest_DELETED - } - - opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value))) - - err = stream.Send(req) - if err != nil { - if errors.Is(err, io.EOF) { - err = nil - } - return nil, err - } - } - - if rows.Error() != nil { - return nil, rows.Error() - } - - opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count)) - return nil, err -} - -func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { - opts.Progress(-1, "migrating library panels...") - panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{ - OrgID: orgId, - Limit: 1000000, - }) - if err != nil { - return nil, err - } - for i, panel := range panels.Items { - meta, err := utils.MetaAccessor(&panel) - if err != nil { - return nil, err - } - body, err := json.Marshal(panel) - if err != nil { - return nil, err - } - - req := &resourcepb.BulkRequest{ - Key: &resourcepb.ResourceKey{ - Namespace: opts.Namespace, - Group: dashboard.GROUP, - Resource: dashboard.LIBRARY_PANEL_RESOURCE, - Name: panel.Name, - }, - Value: body, - Folder: meta.GetFolder(), - Action: resourcepb.BulkRequest_ADDED, - } - if panel.Generation > 1 { - req.Action = resourcepb.BulkRequest_MODIFIED - } - - opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value))) - - err = stream.Send(req) - if err != nil { - if errors.Is(err, io.EOF) { - err = nil - } - return nil, err - } - } - opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items))) - return nil, nil -} diff --git a/pkg/registry/apis/dashboard/legacy/migrate_test.go b/pkg/registry/apis/dashboard/legacy/migrate_test.go deleted file mode 100644 index 5b134cb31dc..00000000000 --- a/pkg/registry/apis/dashboard/legacy/migrate_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package legacy - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/storage/legacysql" - "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" - "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks" -) - -func TestDashboardMigrationQuery(t *testing.T) { - // Test that migration queries use AllowFallback flag correctly - nodb := &legacysql.LegacyDatabaseHelper{ - Table: func(n string) string { - return "grafana." + n - }, - } - - t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) { - // Create a migration query as would be used in actual migration - migrationQuery := &DashboardQuery{ - OrgID: 1, - GetHistory: true, // Migration includes history - AllowFallback: true, // This is the key flag for migration - Order: "ASC", // Migration uses ascending order - } - - // Verify UseHistoryTable returns true (requirement for COALESCE logic) - require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table") - - // Verify the flag is set correctly - require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback") - require.True(t, migrationQuery.GetHistory, "Migration query should get history") - require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order") - }) - - t.Run("Regular history query should not use AllowFallback", func(t *testing.T) { - // Regular history query without migration - historyQuery := &DashboardQuery{ - OrgID: 1, - GetHistory: true, - Order: "DESC", - } - - require.True(t, historyQuery.UseHistoryTable(), "History query should use history table") - require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback") - require.True(t, historyQuery.GetHistory, "History query should get history") - }) - - t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) { - // Test that the SQL template produces COALESCE logic for migration queries - migrationQuery := &DashboardQuery{ - OrgID: 1, - GetHistory: true, - AllowFallback: true, - Order: "ASC", - } - - req := newQueryReq(nodb, migrationQuery) - req.SQLTemplate = mocks.NewTestingSQLTemplate() - - // Execute the template to get the generated SQL - rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req) - require.NoError(t, err) - - sql := rawQuery - - // Verify that COALESCE functions are present in the generated SQL - // These should be used when GetHistory=true AND AllowFallback=true - require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)", - "Migration SQL should contain COALESCE for updated timestamp") - require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)", - "Migration SQL should contain COALESCE for version") - require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)", - "Migration SQL should contain COALESCE for data") - require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)", - "Migration SQL should contain COALESCE for api_version") - require.Contains(t, sql, "COALESCE(dashboard_version.message, '')", - "Migration SQL should contain COALESCE for message with empty string fallback") - - // Verify ORDER BY uses COALESCE as well - require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC", - "Migration SQL should ORDER BY COALESCED created timestamp") - require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC", - "Migration SQL should ORDER BY COALESCED version") - - // Verify it doesn't have the strict history table filter that would exclude NULL version entries - require.NotContains(t, sql, "dashboard_version.id IS NOT NULL", - "Migration SQL should not exclude dashboards without version entries") - }) - - t.Run("Regular history query produces strict SQL", func(t *testing.T) { - // Test that regular history queries still use strict dashboard_version fields - historyQuery := &DashboardQuery{ - OrgID: 1, - GetHistory: true, - Order: "DESC", - } - - req := newQueryReq(nodb, historyQuery) - req.SQLTemplate = mocks.NewTestingSQLTemplate() - - rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req) - require.NoError(t, err) - - sql := rawQuery - - // Verify that direct dashboard_version fields are used (no COALESCE) - require.Contains(t, sql, "dashboard_version.created as updated", - "Regular history SQL should use direct dashboard_version.created") - require.Contains(t, sql, "dashboard_version.version", - "Regular history SQL should use direct dashboard_version.version") - require.Contains(t, sql, "dashboard_version.data", - "Regular history SQL should use direct dashboard_version.data") - - // NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter - // to allow for cases where dashboard_version entries might be missing - - // Should not contain COALESCE functions - require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)", - "Regular history SQL should not contain COALESCE for updated") - }) -} - -func TestMigrateDashboardsConfiguration(t *testing.T) { - // Test the actual migration function configuration - - t.Run("Migration options should configure query correctly", func(t *testing.T) { - // Test the migration configuration as used in real migration - opts := MigrateOptions{ - WithHistory: true, // Migration includes history - } - - // This simulates what happens in migrateDashboards function - expectedQuery := &DashboardQuery{ - OrgID: 1, - Limit: 100000000, - GetHistory: opts.WithHistory, // Should be true - AllowFallback: true, // Should be true for migration - Order: "ASC", // Should be ASC for migration - } - - // Verify the configuration matches what migration sets up - require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory") - require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback") - require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order") - require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit") - - // Verify UseHistoryTable logic - require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table") - }) -} diff --git a/pkg/registry/apis/dashboard/legacy/migration_dashboard_accessor_mock.go b/pkg/registry/apis/dashboard/legacy/migration_dashboard_accessor_mock.go new file mode 100644 index 00000000000..43f2ce05ad1 --- /dev/null +++ b/pkg/registry/apis/dashboard/legacy/migration_dashboard_accessor_mock.go @@ -0,0 +1,279 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package legacy + +import ( + context "context" + + resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb" + mock "github.com/stretchr/testify/mock" +) + +// MockMigrationDashboardAccessor is an autogenerated mock type for the MigrationDashboardAccessor type +type MockMigrationDashboardAccessor struct { + mock.Mock +} + +type MockMigrationDashboardAccessor_Expecter struct { + mock *mock.Mock +} + +func (_m *MockMigrationDashboardAccessor) EXPECT() *MockMigrationDashboardAccessor_Expecter { + return &MockMigrationDashboardAccessor_Expecter{mock: &_m.Mock} +} + +// CountResources provides a mock function with given fields: ctx, opts +func (_m *MockMigrationDashboardAccessor) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { + ret := _m.Called(ctx, opts) + + if len(ret) == 0 { + panic("no return value specified for CountResources") + } + + var r0 *resourcepb.BulkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok { + return rf(ctx, opts) + } + if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok { + r0 = rf(ctx, opts) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*resourcepb.BulkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok { + r1 = rf(ctx, opts) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMigrationDashboardAccessor_CountResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountResources' +type MockMigrationDashboardAccessor_CountResources_Call struct { + *mock.Call +} + +// CountResources is a helper method to define mock.On call +// - ctx context.Context +// - opts MigrateOptions +func (_e *MockMigrationDashboardAccessor_Expecter) CountResources(ctx interface{}, opts interface{}) *MockMigrationDashboardAccessor_CountResources_Call { + return &MockMigrationDashboardAccessor_CountResources_Call{Call: _e.mock.On("CountResources", ctx, opts)} +} + +func (_c *MockMigrationDashboardAccessor_CountResources_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockMigrationDashboardAccessor_CountResources_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(MigrateOptions)) + }) + return _c +} + +func (_c *MockMigrationDashboardAccessor_CountResources_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockMigrationDashboardAccessor_CountResources_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMigrationDashboardAccessor_CountResources_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockMigrationDashboardAccessor_CountResources_Call { + _c.Call.Return(run) + return _c +} + +// MigrateDashboards provides a mock function with given fields: ctx, orgId, opts, stream +func (_m *MockMigrationDashboardAccessor) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + ret := _m.Called(ctx, orgId, opts, stream) + + if len(ret) == 0 { + panic("no return value specified for MigrateDashboards") + } + + var r0 *BlobStoreInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok { + return rf(ctx, orgId, opts, stream) + } + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok { + r0 = rf(ctx, orgId, opts, stream) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*BlobStoreInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok { + r1 = rf(ctx, orgId, opts, stream) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMigrationDashboardAccessor_MigrateDashboards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateDashboards' +type MockMigrationDashboardAccessor_MigrateDashboards_Call struct { + *mock.Call +} + +// MigrateDashboards is a helper method to define mock.On call +// - ctx context.Context +// - orgId int64 +// - opts MigrateOptions +// - stream resourcepb.BulkStore_BulkProcessClient +func (_e *MockMigrationDashboardAccessor_Expecter) MigrateDashboards(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateDashboards_Call { + return &MockMigrationDashboardAccessor_MigrateDashboards_Call{Call: _e.mock.On("MigrateDashboards", ctx, orgId, opts, stream)} +} + +func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateDashboards_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient)) + }) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateDashboards_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateDashboards_Call { + _c.Call.Return(run) + return _c +} + +// MigrateFolders provides a mock function with given fields: ctx, orgId, opts, stream +func (_m *MockMigrationDashboardAccessor) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + ret := _m.Called(ctx, orgId, opts, stream) + + if len(ret) == 0 { + panic("no return value specified for MigrateFolders") + } + + var r0 *BlobStoreInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok { + return rf(ctx, orgId, opts, stream) + } + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok { + r0 = rf(ctx, orgId, opts, stream) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*BlobStoreInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok { + r1 = rf(ctx, orgId, opts, stream) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMigrationDashboardAccessor_MigrateFolders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateFolders' +type MockMigrationDashboardAccessor_MigrateFolders_Call struct { + *mock.Call +} + +// MigrateFolders is a helper method to define mock.On call +// - ctx context.Context +// - orgId int64 +// - opts MigrateOptions +// - stream resourcepb.BulkStore_BulkProcessClient +func (_e *MockMigrationDashboardAccessor_Expecter) MigrateFolders(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateFolders_Call { + return &MockMigrationDashboardAccessor_MigrateFolders_Call{Call: _e.mock.On("MigrateFolders", ctx, orgId, opts, stream)} +} + +func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateFolders_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient)) + }) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateFolders_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateFolders_Call { + _c.Call.Return(run) + return _c +} + +// MigrateLibraryPanels provides a mock function with given fields: ctx, orgId, opts, stream +func (_m *MockMigrationDashboardAccessor) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + ret := _m.Called(ctx, orgId, opts, stream) + + if len(ret) == 0 { + panic("no return value specified for MigrateLibraryPanels") + } + + var r0 *BlobStoreInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok { + return rf(ctx, orgId, opts, stream) + } + if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok { + r0 = rf(ctx, orgId, opts, stream) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*BlobStoreInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok { + r1 = rf(ctx, orgId, opts, stream) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMigrationDashboardAccessor_MigrateLibraryPanels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateLibraryPanels' +type MockMigrationDashboardAccessor_MigrateLibraryPanels_Call struct { + *mock.Call +} + +// MigrateLibraryPanels is a helper method to define mock.On call +// - ctx context.Context +// - orgId int64 +// - opts MigrateOptions +// - stream resourcepb.BulkStore_BulkProcessClient +func (_e *MockMigrationDashboardAccessor_Expecter) MigrateLibraryPanels(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call { + return &MockMigrationDashboardAccessor_MigrateLibraryPanels_Call{Call: _e.mock.On("MigrateLibraryPanels", ctx, orgId, opts, stream)} +} + +func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient)) + }) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call { + _c.Call.Return(run) + return _c +} + +// NewMockMigrationDashboardAccessor creates a new instance of MockMigrationDashboardAccessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockMigrationDashboardAccessor(t interface { + mock.TestingT + Cleanup(func()) +}) *MockMigrationDashboardAccessor { + mock := &MockMigrationDashboardAccessor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go index 3d157e09489..7c17ec8b6aa 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go @@ -4,7 +4,9 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" + "io" "strconv" "strings" "sync" @@ -13,6 +15,7 @@ import ( "go.opentelemetry.io/otel" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" claims "github.com/grafana/authlib/types" @@ -20,6 +23,7 @@ import ( dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1" dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" "github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion" + folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" @@ -35,16 +39,30 @@ import ( "github.com/grafana/grafana/pkg/services/librarypanels" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/search/sort" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/storage/legacysql" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/grafana/grafana/pkg/storage/unified/resourcepb" "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" ) var ( - _ DashboardAccess = (*dashboardSqlAccess)(nil) - tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy") + tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy") ) +type MigrateOptions struct { + Namespace string + Resources []schema.GroupResource + WithHistory bool // only applies to dashboards + OnlyCount bool // just count the values + Progress func(count int, msg string) +} + +type BlobStoreInfo struct { + Count int64 + Size int64 +} + type dashboardRow struct { // The numeric version for this dashboard RV int64 @@ -63,8 +81,9 @@ type dashboardRow struct { type dashboardSqlAccess struct { sql legacysql.LegacyDatabaseProvider namespacer request.NamespaceMapper - provisioning provisioning.ProvisioningService + provisioning provisioning.StubProvisioningService + // TODO: consider enabling this by default for on-prem migrations invalidDashboardParseFallbackEnabled bool // Use for writing (not reading) @@ -73,7 +92,7 @@ type dashboardSqlAccess struct { dashboardPermissionSvc accesscontrol.DashboardPermissionsService accessControl accesscontrol.AccessControl - libraryPanelSvc librarypanels.Service + libraryPanelSvc librarypanels.Service // only used for save dashboard // Typically one... the server wrapper subscribers []chan *resource.WrittenEvent @@ -81,7 +100,27 @@ type dashboardSqlAccess struct { log log.Logger } -func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider, +// ProvideMigratorDashboardAccessor creates a DashboardAccess specifically for migration purposes. +// This provider is used by Wire DI and only includes the minimal dependencies needed for migrations. +func ProvideMigratorDashboardAccessor( + sql legacysql.LegacyDatabaseProvider, + provisioning provisioning.StubProvisioningService, + accessControl accesscontrol.AccessControl, + features featuremgmt.FeatureToggles, +) MigrationDashboardAccessor { + return &dashboardSqlAccess{ + sql: sql, + namespacer: claims.OrgNamespaceFormatter, + dashStore: nil, // not needed for migration + provisioning: provisioning, + dashboardPermissionSvc: nil, // not needed for migration + libraryPanelSvc: nil, // not needed for migration + accessControl: accessControl, + invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled), + } +} + +func NewDashboardSQLAccess(sql legacysql.LegacyDatabaseProvider, namespacer request.NamespaceMapper, dashStore dashboards.Store, provisioning provisioning.ProvisioningService, @@ -90,7 +129,7 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider, dashboardPermissionSvc accesscontrol.DashboardPermissionsService, accessControl accesscontrol.AccessControl, features featuremgmt.FeatureToggles, -) DashboardAccess { +) *dashboardSqlAccess { dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter) return &dashboardSqlAccess{ sql: sql, @@ -101,7 +140,6 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider, dashboardPermissionSvc: dashboardPermissionSvc, libraryPanelSvc: libraryPanelSvc, accessControl: accessControl, - log: log.New("dashboard.legacysql"), invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled), } } @@ -149,6 +187,290 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD }, err } +// CountResources counts resources without migrating them +func (a *dashboardSqlAccess) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) { + sql, err := a.sql(ctx) + if err != nil { + return nil, err + } + ns, err := claims.ParseNamespace(opts.Namespace) + if err != nil { + return nil, err + } + orgId := ns.OrgID + rsp := &resourcepb.BulkResponse{} + err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { + for _, res := range opts.Resources { + switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { + case "folder.grafana.app/folders": + summary := &resourcepb.BulkResponse_Summary{} + summary.Group = folders.GROUP + summary.Group = folders.RESOURCE + _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+ + " WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count) + rsp.Summary = append(rsp.Summary, summary) + + case "dashboard.grafana.app/librarypanels": + summary := &resourcepb.BulkResponse_Summary{} + summary.Group = dashboardV1.GROUP + summary.Resource = dashboardV1.LIBRARY_PANEL_RESOURCE + _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+ + " WHERE org_id=?", orgId).Get(&summary.Count) + rsp.Summary = append(rsp.Summary, summary) + + case "dashboard.grafana.app/dashboards": + summary := &resourcepb.BulkResponse_Summary{} + summary.Group = dashboardV1.GROUP + summary.Resource = dashboardV1.DASHBOARD_RESOURCE + rsp.Summary = append(rsp.Summary, summary) + + _, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+ + " WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count) + if err != nil { + return err + } + + // Also count history + _, err = sess.SQL(`SELECT COUNT(*) + FROM `+sql.Table("dashboard_version")+` as dv + JOIN `+sql.Table("dashboard")+` as dd + ON dd.id = dv.dashboard_id + WHERE org_id=?`, orgId).Get(&summary.History) + } + if err != nil { + return err + } + } + return nil + }) + return rsp, nil +} + +// MigrateDashboards handles the dashboard migration logic +func (a *dashboardSqlAccess) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + query := &DashboardQuery{ + OrgID: orgId, + Limit: 100000000, + GetHistory: opts.WithHistory, // include history + AllowFallback: true, // allow fallback to dashboard table during migration + Order: "ASC", // oldest first + } + + blobs := &BlobStoreInfo{} + sql, err := a.sql(ctx) + if err != nil { + return blobs, err + } + + opts.Progress(-1, "migrating dashboards...") + rows, err := a.getRows(ctx, sql, query) + if rows != nil { + defer func() { + _ = rows.Close() + }() + } + if err != nil { + return blobs, err + } + + // Now send each dashboard + for i := 1; rows.Next(); i++ { + dash := rows.row.Dash + if dash.APIVersion == "" { + dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboardV1.GROUP) + } + dash.SetNamespace(opts.Namespace) + dash.SetResourceVersion("") // it will be filled in by the backend + + body, err := json.Marshal(dash) + if err != nil { + err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err) + return blobs, err + } + + req := &resourcepb.BulkRequest{ + Key: &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: dashboardV1.GROUP, + Resource: dashboardV1.DASHBOARD_RESOURCE, + Name: rows.Name(), + }, + Value: body, + Folder: rows.row.FolderUID, + Action: resourcepb.BulkRequest_ADDED, + } + if dash.Generation > 1 { + req.Action = resourcepb.BulkRequest_MODIFIED + } else if dash.Generation < 0 { + req.Action = resourcepb.BulkRequest_DELETED + } + + opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count)) + + err = stream.Send(req) + if err != nil { + if errors.Is(err, io.EOF) { + opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i)) + err = nil + } + return blobs, err + } + } + + if len(rows.rejected) > 0 { + for _, row := range rows.rejected { + id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID] + a.log.Warn("rejected dashboard", + "namespace", opts.Namespace, + "dashboard", row.Dash.Name, + "uid", row.Dash.UID, + "id", id, + "version", row.Dash.Generation, + ) + opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name)) + } + } + + if rows.Error() != nil { + return blobs, rows.Error() + } + + opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count)) + return blobs, err +} + +// MigrateFolders handles the folder migration logic +func (a *dashboardSqlAccess) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + query := &DashboardQuery{ + OrgID: orgId, + Limit: 100000000, + GetFolders: true, + Order: "ASC", + } + + sql, err := a.sql(ctx) + if err != nil { + return nil, err + } + + opts.Progress(-1, "migrating folders...") + rows, err := a.getRows(ctx, sql, query) + if rows != nil { + defer func() { + _ = rows.Close() + }() + } + if err != nil { + return nil, err + } + + // Now send each dashboard + for i := 1; rows.Next(); i++ { + dash := rows.row.Dash + dash.APIVersion = "folder.grafana.app/v1beta1" + dash.Kind = "Folder" + dash.SetNamespace(opts.Namespace) + dash.SetResourceVersion("") // it will be filled in by the backend + + spec := map[string]any{ + "title": dash.Spec.Object["title"], + } + description := dash.Spec.Object["description"] + if description != nil { + spec["description"] = description + } + dash.Spec.Object = spec + + body, err := json.Marshal(dash) + if err != nil { + return nil, err + } + + req := &resourcepb.BulkRequest{ + Key: &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: "folder.grafana.app", + Resource: "folders", + Name: rows.Name(), + }, + Value: body, + Folder: rows.row.FolderUID, + Action: resourcepb.BulkRequest_ADDED, + } + if dash.Generation > 1 { + req.Action = resourcepb.BulkRequest_MODIFIED + } else if dash.Generation < 0 { + req.Action = resourcepb.BulkRequest_DELETED + } + + opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value))) + + err = stream.Send(req) + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + return nil, err + } + } + + if rows.Error() != nil { + return nil, rows.Error() + } + + opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count)) + return nil, err +} + +// MigrateLibraryPanels handles the library panel migration logic +func (a *dashboardSqlAccess) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) { + opts.Progress(-1, "migrating library panels...") + panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{ + OrgID: orgId, + Limit: 1000000, + }) + if err != nil { + return nil, err + } + for i, panel := range panels.Items { + meta, err := utils.MetaAccessor(&panel) + if err != nil { + return nil, err + } + body, err := json.Marshal(panel) + if err != nil { + return nil, err + } + + req := &resourcepb.BulkRequest{ + Key: &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: dashboardV1.GROUP, + Resource: dashboardV1.LIBRARY_PANEL_RESOURCE, + Name: panel.Name, + }, + Value: body, + Folder: meta.GetFolder(), + Action: resourcepb.BulkRequest_ADDED, + } + if panel.Generation > 1 { + req.Action = resourcepb.BulkRequest_MODIFIED + } + + opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value))) + + err = stream.Send(req) + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + return nil, err + } + } + opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items))) + return nil, nil +} + var _ resource.ListIterator = (*rowsWrapper)(nil) type rowsWrapper struct { diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go index e33841c044c..b142b144800 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go @@ -21,6 +21,9 @@ import ( "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/storage/legacysql" + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks" ) func TestScanRow(t *testing.T) { @@ -529,3 +532,146 @@ func TestParseLibraryPanelRow(t *testing.T) { require.Nil(t, updatedTimestamp) }) } + +func TestDashboardMigrationQuery(t *testing.T) { + // Test that migration queries use AllowFallback flag correctly + nodb := &legacysql.LegacyDatabaseHelper{ + Table: func(n string) string { + return "grafana." + n + }, + } + + t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) { + // Create a migration query as would be used in actual migration + migrationQuery := &DashboardQuery{ + OrgID: 1, + GetHistory: true, // Migration includes history + AllowFallback: true, // This is the key flag for migration + Order: "ASC", // Migration uses ascending order + } + + // Verify UseHistoryTable returns true (requirement for COALESCE logic) + require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table") + + // Verify the flag is set correctly + require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback") + require.True(t, migrationQuery.GetHistory, "Migration query should get history") + require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order") + }) + + t.Run("Regular history query should not use AllowFallback", func(t *testing.T) { + // Regular history query without migration + historyQuery := &DashboardQuery{ + OrgID: 1, + GetHistory: true, + Order: "DESC", + } + + require.True(t, historyQuery.UseHistoryTable(), "History query should use history table") + require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback") + require.True(t, historyQuery.GetHistory, "History query should get history") + }) + + t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) { + // Test that the SQL template produces COALESCE logic for migration queries + migrationQuery := &DashboardQuery{ + OrgID: 1, + GetHistory: true, + AllowFallback: true, + Order: "ASC", + } + + req := newQueryReq(nodb, migrationQuery) + req.SQLTemplate = mocks.NewTestingSQLTemplate() + + // Execute the template to get the generated SQL + rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req) + require.NoError(t, err) + + sql := rawQuery + + // Verify that COALESCE functions are present in the generated SQL + // These should be used when GetHistory=true AND AllowFallback=true + require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)", + "Migration SQL should contain COALESCE for updated timestamp") + require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)", + "Migration SQL should contain COALESCE for version") + require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)", + "Migration SQL should contain COALESCE for data") + require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)", + "Migration SQL should contain COALESCE for api_version") + require.Contains(t, sql, "COALESCE(dashboard_version.message, '')", + "Migration SQL should contain COALESCE for message with empty string fallback") + + // Verify ORDER BY uses COALESCE as well + require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC", + "Migration SQL should ORDER BY COALESCED created timestamp") + require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC", + "Migration SQL should ORDER BY COALESCED version") + + // Verify it doesn't have the strict history table filter that would exclude NULL version entries + require.NotContains(t, sql, "dashboard_version.id IS NOT NULL", + "Migration SQL should not exclude dashboards without version entries") + }) + + t.Run("Regular history query produces strict SQL", func(t *testing.T) { + // Test that regular history queries still use strict dashboard_version fields + historyQuery := &DashboardQuery{ + OrgID: 1, + GetHistory: true, + Order: "DESC", + } + + req := newQueryReq(nodb, historyQuery) + req.SQLTemplate = mocks.NewTestingSQLTemplate() + + rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req) + require.NoError(t, err) + + sql := rawQuery + + // Verify that direct dashboard_version fields are used (no COALESCE) + require.Contains(t, sql, "dashboard_version.created as updated", + "Regular history SQL should use direct dashboard_version.created") + require.Contains(t, sql, "dashboard_version.version", + "Regular history SQL should use direct dashboard_version.version") + require.Contains(t, sql, "dashboard_version.data", + "Regular history SQL should use direct dashboard_version.data") + + // NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter + // to allow for cases where dashboard_version entries might be missing + + // Should not contain COALESCE functions + require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)", + "Regular history SQL should not contain COALESCE for updated") + }) +} + +func TestMigrateDashboardsConfiguration(t *testing.T) { + // Test the actual migration function configuration + + t.Run("Migration options should configure query correctly", func(t *testing.T) { + // Test the migration configuration as used in real migration + opts := MigrateOptions{ + WithHistory: true, // Migration includes history + } + + // This simulates what happens in migrateDashboards function + expectedQuery := &DashboardQuery{ + OrgID: 1, + Limit: 100000000, + GetHistory: opts.WithHistory, // Should be true + AllowFallback: true, // Should be true for migration + Order: "ASC", // Should be ASC for migration + } + + // Verify the configuration matches what migration sets up + require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory") + require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback") + require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order") + require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit") + + // Verify UseHistoryTable logic + require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table") + }) +} diff --git a/pkg/registry/apis/dashboard/legacy/types.go b/pkg/registry/apis/dashboard/legacy/types.go index 4ffa7b80863..31d08af16d8 100644 --- a/pkg/registry/apis/dashboard/legacy/types.go +++ b/pkg/registry/apis/dashboard/legacy/types.go @@ -55,10 +55,9 @@ type LibraryPanelQuery struct { LastID int64 } -type DashboardAccess interface { +type DashboardAccessor interface { resource.StorageBackend resourcepb.ResourceIndexServer - LegacyMigrator GetDashboard(ctx context.Context, orgId int64, uid string, version int64) (*dashboardV1.Dashboard, int64, error) SaveDashboard(ctx context.Context, orgId int64, dash *dashboardV1.Dashboard, failOnExisting bool) (*dashboardV1.Dashboard, bool, error) @@ -67,3 +66,12 @@ type DashboardAccess interface { // Get a typed list GetLibraryPanels(ctx context.Context, query LibraryPanelQuery) (*dashboardV0.LibraryPanelList, error) } + +//go:generate mockery --name MigrationDashboardAccessor --structname MockMigrationDashboardAccessor --inpackage --filename migration_dashboard_accessor_mock.go --with-expecter +type MigrationDashboardAccessor interface { + // Migration helper methods - these support the separate LegacyMigrator + CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) + MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) + MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) + MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) +} diff --git a/pkg/registry/apis/dashboard/legacy_storage.go b/pkg/registry/apis/dashboard/legacy_storage.go index d51bea21bf9..1439574efe2 100644 --- a/pkg/registry/apis/dashboard/legacy_storage.go +++ b/pkg/registry/apis/dashboard/legacy_storage.go @@ -22,7 +22,7 @@ import ( ) type DashboardStorage struct { - Access legacy.DashboardAccess + Access legacy.DashboardAccessor DashboardService dashboards.DashboardService } diff --git a/pkg/registry/apis/dashboard/libary_panel.go b/pkg/registry/apis/dashboard/libary_panel.go index aa2b2705bb1..f170831da2b 100644 --- a/pkg/registry/apis/dashboard/libary_panel.go +++ b/pkg/registry/apis/dashboard/libary_panel.go @@ -30,7 +30,7 @@ var ( ) type LibraryPanelStore struct { - Access legacy.DashboardAccess + Access legacy.DashboardAccessor ResourceInfo utils.ResourceInfo service libraryelements.Service } diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index f784fc9c014..fb42543de33 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -169,7 +169,7 @@ func RegisterAPIService( publicDashboardService: publicDashboardService, legacy: &DashboardStorage{ - Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features), + Access: legacy.NewDashboardSQLAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features), DashboardService: dashboardService, }, } diff --git a/pkg/registry/apis/dashboard/sub_dto.go b/pkg/registry/apis/dashboard/sub_dto.go index f2e9bc0b6df..04a14387fc7 100644 --- a/pkg/registry/apis/dashboard/sub_dto.go +++ b/pkg/registry/apis/dashboard/sub_dto.go @@ -30,7 +30,7 @@ type dtoBuilder = func(dashboard runtime.Object, access *dashboard.DashboardAcce // The DTO returns everything the UI needs in a single request type DTOConnector struct { getter rest.Getter - legacy legacy.DashboardAccess + legacy legacy.DashboardAccessor unified resource.ResourceClient largeObjects apistore.LargeObjectSupport accessControl accesscontrol.AccessControl @@ -42,7 +42,7 @@ type DTOConnector struct { func NewDTOConnector( getter rest.Getter, largeObjects apistore.LargeObjectSupport, - legacyAccess legacy.DashboardAccess, + legacyAccess legacy.DashboardAccessor, resourceClient resource.ResourceClient, accessControl accesscontrol.AccessControl, scheme *runtime.Scheme, diff --git a/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources.go b/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources.go index b7b4bb45a4f..48add96ee16 100644 --- a/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources.go +++ b/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources/signature" + unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations" "github.com/grafana/grafana/pkg/storage/unified/parquet" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" @@ -29,7 +30,7 @@ type LegacyResourcesMigrator interface { type legacyResourcesMigrator struct { repositoryResources resources.RepositoryResourcesFactory parsers resources.ParserFactory - legacyMigrator legacy.LegacyMigrator + dashboardAccess legacy.MigrationDashboardAccessor signerFactory signature.SignerFactory clients resources.ClientFactory exportFn export.ExportFn @@ -38,7 +39,7 @@ type legacyResourcesMigrator struct { func NewLegacyResourcesMigrator( repositoryResources resources.RepositoryResourcesFactory, parsers resources.ParserFactory, - legacyMigrator legacy.LegacyMigrator, + dashboardAccess legacy.MigrationDashboardAccessor, signerFactory signature.SignerFactory, clients resources.ClientFactory, exportFn export.ExportFn, @@ -46,7 +47,7 @@ func NewLegacyResourcesMigrator( return &legacyResourcesMigrator{ repositoryResources: repositoryResources, parsers: parsers, - legacyMigrator: legacyMigrator, + dashboardAccess: dashboardAccess, signerFactory: signerFactory, clients: clients, exportFn: exportFn, @@ -94,7 +95,7 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea reader := newLegacyResourceMigrator( rw, - m.legacyMigrator, + m.dashboardAccess, parser, repositoryResources, progress, @@ -113,21 +114,21 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea } type legacyResourceResourceMigrator struct { - repo repository.ReaderWriter - legacy legacy.LegacyMigrator - parser resources.Parser - progress jobs.JobProgressRecorder - namespace string - kind schema.GroupResource - options provisioning.MigrateJobOptions - resources resources.RepositoryResources - signer signature.Signer - history map[string]string // UID >> file path + repo repository.ReaderWriter + dashboardAccess legacy.MigrationDashboardAccessor + parser resources.Parser + progress jobs.JobProgressRecorder + namespace string + kind schema.GroupResource + options provisioning.MigrateJobOptions + resources resources.RepositoryResources + signer signature.Signer + history map[string]string // UID >> file path } func newLegacyResourceMigrator( repo repository.ReaderWriter, - legacy legacy.LegacyMigrator, + dashboardAccess legacy.MigrationDashboardAccessor, parser resources.Parser, resources resources.RepositoryResources, progress jobs.JobProgressRecorder, @@ -141,16 +142,16 @@ func newLegacyResourceMigrator( history = make(map[string]string) } return &legacyResourceResourceMigrator{ - repo: repo, - legacy: legacy, - parser: parser, - progress: progress, - options: options, - namespace: namespace, - kind: kind, - resources: resources, - signer: signer, - history: history, + repo: repo, + dashboardAccess: dashboardAccess, + parser: parser, + progress: progress, + options: options, + namespace: namespace, + kind: kind, + resources: resources, + signer: signer, + history: history, } } @@ -225,14 +226,21 @@ func (r *legacyResourceResourceMigrator) Write(ctx context.Context, key *resourc func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error { r.progress.SetMessage(ctx, fmt.Sprintf("migrate %s resource", r.kind.Resource)) + + // Create a parquet migrator with this instance as the BulkResourceWriter + parquetClient := parquet.NewBulkResourceWriterClient(r) + migrator := unifiedmigrations.ProvideUnifiedMigratorParquet( + r.dashboardAccess, + parquetClient, + ) + opts := legacy.MigrateOptions{ Namespace: r.namespace, WithHistory: r.options.History, Resources: []schema.GroupResource{r.kind}, - Store: parquet.NewBulkResourceWriterClient(r), OnlyCount: true, // first get the count } - stats, err := r.legacy.Migrate(ctx, opts) + stats, err := migrator.Migrate(ctx, opts) if err != nil { return fmt.Errorf("unable to count legacy items %w", err) } @@ -248,7 +256,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error { } opts.OnlyCount = false // this time actually write - _, err = r.legacy.Migrate(ctx, opts) + _, err = migrator.Migrate(ctx, opts) if err != nil { return fmt.Errorf("migrate legacy %s: %w", r.kind.Resource, err) } diff --git a/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources_test.go b/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources_test.go index 89077848f25..7f4318639c2 100644 --- a/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources_test.go +++ b/pkg/registry/apis/provisioning/jobs/migrate/legacy_resources_test.go @@ -93,8 +93,8 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { mockRepoResourcesFactory.On("Client", mock.Anything, mock.Anything). Return(mockRepoResources, nil) - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{}, errors.New("legacy migrator error")) @@ -115,7 +115,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { migrator := NewLegacyResourcesMigrator( mockRepoResourcesFactory, mockParserFactory, - mockLegacyMigrator, + mockDashboardAccess, signerFactory, mockClientFactory, mockExportFn.Execute, @@ -136,7 +136,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { mockParserFactory.AssertExpectations(t) mockRepoResourcesFactory.AssertExpectations(t) - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) mockExportFn.AssertExpectations(t) mockClientFactory.AssertExpectations(t) @@ -308,22 +308,18 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { History: true, }).Return(mockSigner, nil) - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + // Mock CountResources for the count phase + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + })).Return(&resourcepb.BulkResponse{}, nil).Once() + // Mock MigrateDashboards for the actual migration phase (dashboards resource) + mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return !opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{ - Summary: []*resourcepb.BulkResponse_Summary{ - { - Group: "test.grafana.app", - Resource: "tests", - Count: 10, - History: 5, - }, - }, - }, nil).Once() // Migration phase + }), mock.Anything).Return(&legacy.BlobStoreInfo{ + Count: 10, + Size: 5, + }, nil).Once() mockClients := resources.NewMockResourceClients(t) mockClientFactory := resources.NewMockClientFactory(t) @@ -339,7 +335,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { migrator := NewLegacyResourcesMigrator( mockRepoResourcesFactory, mockParserFactory, - mockLegacyMigrator, + mockDashboardAccess, mockSignerFactory, mockClientFactory, mockExportFn.Execute, @@ -362,7 +358,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) { mockParserFactory.AssertExpectations(t) mockRepoResourcesFactory.AssertExpectations(t) - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) mockClientFactory.AssertExpectations(t) mockExportFn.AssertExpectations(t) progress.AssertExpectations(t) @@ -742,8 +738,8 @@ func TestLegacyResourceResourceMigrator_Write(t *testing.T) { func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) { t.Run("should fail when legacy migrate count fails", func(t *testing.T) { - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{}, errors.New("count error")) @@ -752,7 +748,7 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) { migrator := newLegacyResourceMigrator( nil, - mockLegacyMigrator, + mockDashboardAccess, nil, nil, progress, @@ -766,89 +762,91 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "unable to count legacy items") - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) }) t.Run("should fail when legacy migrate write fails", func(t *testing.T) { - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + // For test-resources GroupResource, we don't know which method it will call, but since it's not dashboards/folders/librarypanels, + // the Migrate will fail trying to map the resource type. Let's make it dashboards for this test. + mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return !opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{}, errors.New("write error")).Once() // Write phase + }), mock.Anything).Return(nil, errors.New("write error")).Once() // Write phase progress := jobs.NewMockJobProgressRecorder(t) progress.On("SetMessage", mock.Anything, mock.Anything).Return() migrator := newLegacyResourceMigrator( nil, - mockLegacyMigrator, + mockDashboardAccess, nil, nil, progress, provisioning.MigrateJobOptions{}, "test-namespace", - schema.GroupResource{Group: "test.grafana.app", Resource: "test-resources"}, + schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}, signature.NewGrafanaSigner(), ) err := migrator.Migrate(context.Background()) require.Error(t, err) - require.Contains(t, err.Error(), "migrate legacy test-resources: write error") + require.Contains(t, err.Error(), "migrate legacy dashboards: write error") - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) }) t.Run("should successfully migrate resource", func(t *testing.T) { - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return !opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase + }), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase progress := jobs.NewMockJobProgressRecorder(t) progress.On("SetMessage", mock.Anything, mock.Anything).Return() migrator := newLegacyResourceMigrator( nil, - mockLegacyMigrator, + mockDashboardAccess, nil, nil, progress, provisioning.MigrateJobOptions{}, "test-namespace", - schema.GroupResource{Group: "test.grafana.app", Resource: "tests"}, + schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}, signature.NewGrafanaSigner(), ) err := migrator.Migrate(context.Background()) require.NoError(t, err) - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) }) t.Run("should set total to history if history is greater than count", func(t *testing.T) { - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{ Summary: []*resourcepb.BulkResponse_Summary{ { - Group: "test.grafana.app", - Resource: "tests", + Group: "dashboard.grafana.app", + Resource: "dashboards", Count: 1, History: 100, }, }, }, nil).Once() // Count phase - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return !opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase + }), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase progress := jobs.NewMockJobProgressRecorder(t) progress.On("SetMessage", mock.Anything, mock.Anything).Return() @@ -856,39 +854,39 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) { migrator := newLegacyResourceMigrator( nil, - mockLegacyMigrator, + mockDashboardAccess, nil, nil, progress, provisioning.MigrateJobOptions{}, "test-namespace", - schema.GroupResource{Group: "test.grafana.app", Resource: "tests"}, + schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}, signature.NewGrafanaSigner(), ) err := migrator.Migrate(context.Background()) require.NoError(t, err) - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) }) t.Run("should set total to count if history is less than count", func(t *testing.T) { - mockLegacyMigrator := legacy.NewMockLegacyMigrator(t) - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t) + mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return opts.OnlyCount && opts.Namespace == "test-namespace" })).Return(&resourcepb.BulkResponse{ Summary: []*resourcepb.BulkResponse_Summary{ { - Group: "test.grafana.app", - Resource: "tests", + Group: "dashboard.grafana.app", + Resource: "dashboards", Count: 200, History: 1, }, }, }, nil).Once() // Count phase - mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { + mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool { return !opts.OnlyCount && opts.Namespace == "test-namespace" - })).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase + }), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase progress := jobs.NewMockJobProgressRecorder(t) progress.On("SetMessage", mock.Anything, mock.Anything).Return() @@ -897,20 +895,20 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) { migrator := newLegacyResourceMigrator( nil, - mockLegacyMigrator, + mockDashboardAccess, nil, nil, progress, provisioning.MigrateJobOptions{}, "test-namespace", - schema.GroupResource{Group: "test.grafana.app", Resource: "tests"}, + schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}, signer, ) err := migrator.Migrate(context.Background()) require.NoError(t, err) - mockLegacyMigrator.AssertExpectations(t) + mockDashboardAccess.AssertExpectations(t) progress.AssertExpectations(t) }) } diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index c30b43eeb60..d18fc1156a8 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -60,6 +60,7 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/grafana/grafana/pkg/storage/unified/migrations" "github.com/grafana/grafana/pkg/storage/unified/resource" ) @@ -109,7 +110,7 @@ type APIBuilder struct { jobHistoryConfig *JobHistoryConfig jobHistoryLoki *jobs.LokiJobHistory resourceLister resources.ResourceLister - legacyMigrator legacy.LegacyMigrator + dashboardAccess legacy.MigrationDashboardAccessor storageStatus dualwrite.Service unified resource.ResourceClient repoFactory repository.Factory @@ -135,7 +136,7 @@ func NewAPIBuilder( features featuremgmt.FeatureToggles, unified resource.ResourceClient, configProvider apiserver.RestConfigProvider, - legacyMigrator legacy.LegacyMigrator, + dashboardAccess legacy.MigrationDashboardAccessor, storageStatus dualwrite.Service, usageStats usagestats.Service, access authlib.AccessChecker, @@ -158,6 +159,7 @@ func NewAPIBuilder( clients = resources.NewClientFactory(configProvider) } parsers := resources.NewParserFactory(clients) + legacyMigrator := migrations.ProvideUnifiedMigrator(dashboardAccess, unified) resourceLister := resources.NewResourceListerForMigrations(unified, legacyMigrator, storageStatus) b := &APIBuilder{ @@ -170,7 +172,7 @@ func NewAPIBuilder( parsers: parsers, repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister), resourceLister: resourceLister, - legacyMigrator: legacyMigrator, + dashboardAccess: dashboardAccess, storageStatus: storageStatus, unified: unified, access: access, @@ -234,7 +236,7 @@ func RegisterAPIService( client resource.ResourceClient, // implements resource.RepositoryClient configProvider apiserver.RestConfigProvider, access authlib.AccessClient, - legacyMigrator legacy.LegacyMigrator, + dashboardAccess legacy.MigrationDashboardAccessor, storageStatus dualwrite.Service, usageStats usagestats.Service, tracer tracing.Tracer, @@ -258,7 +260,7 @@ func RegisterAPIService( features, client, configProvider, - legacyMigrator, storageStatus, + dashboardAccess, storageStatus, usageStats, access, tracer, @@ -722,7 +724,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH legacyResources := migrate.NewLegacyResourcesMigrator( b.repositoryResources, b.parsers, - b.legacyMigrator, + b.dashboardAccess, signerFactory, b.clients, export.ExportAll, @@ -1241,8 +1243,9 @@ func (b *APIBuilder) tryRunningOnlyUnifiedStorage() error { return nil } - // Count how many things exist - rsp, err := b.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{ + // Count how many things exist - create a migrator on-demand for this + legacyMigrator := migrations.ProvideUnifiedMigrator(b.dashboardAccess, b.unified) + rsp, err := legacyMigrator.Migrate(ctx, legacy.MigrateOptions{ Namespace: "default", // FIXME! this works for single org, but need to check multi-org Resources: []schema.GroupResource{{ Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE, diff --git a/pkg/registry/apis/provisioning/resources/object.go b/pkg/registry/apis/provisioning/resources/object.go index 4d08de5c05c..0a016786410 100644 --- a/pkg/registry/apis/provisioning/resources/object.go +++ b/pkg/registry/apis/provisioning/resources/object.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/grafana/grafana/pkg/storage/unified/migrations" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" ) @@ -30,9 +31,9 @@ type ResourceStore interface { } type ResourceListerFromSearch struct { - store ResourceStore - legacyMigrator legacy.LegacyMigrator - storageStatus dualwrite.Service + store ResourceStore + migrator migrations.UnifiedMigrator + storageStatus dualwrite.Service } func NewResourceLister(store ResourceStore) ResourceLister { @@ -42,13 +43,13 @@ func NewResourceLister(store ResourceStore) ResourceLister { // FIXME: the logic about migration and storage should probably be separated from this func NewResourceListerForMigrations( store ResourceStore, - legacyMigrator legacy.LegacyMigrator, + migrator migrations.UnifiedMigrator, storageStatus dualwrite.Service, ) ResourceLister { return &ResourceListerFromSearch{ - store: store, - legacyMigrator: legacyMigrator, - storageStatus: storageStatus, + store: store, + migrator: migrator, + storageStatus: storageStatus, } } @@ -133,8 +134,8 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit } // Get the stats based on what a migration could support - if o.storageStatus != nil && o.legacyMigrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) { - rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{ + if o.storageStatus != nil && o.migrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) { + rsp, err := o.migrator.Migrate(ctx, legacy.MigrateOptions{ Namespace: namespace, Resources: []schema.GroupResource{{ Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE, diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 5f997875985..e80bcb720c9 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -48,7 +48,6 @@ import ( "github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl" "github.com/grafana/grafana/pkg/services/team/teamapi" "github.com/grafana/grafana/pkg/services/updatemanager" - unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations" ) func ProvideBackgroundServiceRegistry( @@ -74,7 +73,6 @@ func ProvideBackgroundServiceRegistry( dashboardServiceImpl *service.DashboardServiceImpl, secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker, fixedRolesLoader *accesscontrol.FixedRolesLoader, - unifiedStorageMigrationProvider unifiedmigrations.UnifiedStorageMigrationProvider, // Need to make sure these are initialized, is there a better place to put them? _ dashboardsnapshots.Service, _ serviceaccounts.Service, @@ -91,7 +89,6 @@ func ProvideBackgroundServiceRegistry( notifications, rendering, tokenService, - unifiedStorageMigrationProvider, provisioning, grafanaUpdateChecker, pluginsUpdateChecker, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 2fe6db16b31..0d4bb10c0b5 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -128,6 +128,7 @@ import ( "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginaccesscontrol" "github.com/grafana/grafana/pkg/services/preference/prefimpl" promTypeMigration "github.com/grafana/grafana/pkg/services/promtypemigration" + "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/publicdashboards" publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api" publicdashboardsStore "github.com/grafana/grafana/pkg/services/publicdashboards/database" @@ -238,7 +239,9 @@ var wireBasicSet = wire.NewSet( uss.ProvideService, wire.Bind(new(usagestats.Service), new(*uss.UsageStats)), validator.ProvideService, - legacy.ProvideLegacyMigrator, + provisioning.ProvideStubProvisioningService, + legacy.ProvideMigratorDashboardAccessor, + unifiedmigrations.ProvideUnifiedMigrator, pluginsintegration.WireSet, pluginDashboards.ProvideFileStoreManager, wire.Bind(new(pluginDashboards.FileStore), new(*pluginDashboards.FileStoreManager)), @@ -466,8 +469,7 @@ var wireBasicSet = wire.NewSet( // Unified storage resource.ProvideStorageMetrics, resource.ProvideIndexMetrics, - unifiedmigrations.ProvideUnifiedStorageMigrationProvider, - wire.Bind(new(unifiedmigrations.UnifiedStorageMigrationProvider), new(*unifiedmigrations.UnifiedStorageMigrationProviderImpl)), + unifiedmigrations.ProvideUnifiedStorageMigrationService, // Kubernetes API server grafanaapiserver.WireSet, apiregistry.WireSet, diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index 91a1e50a224..0cb27791c19 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -219,7 +219,7 @@ import ( "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/database" kvstore2 "github.com/grafana/grafana/pkg/services/secrets/kvstore" - migrations2 "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations" + migrations3 "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations" "github.com/grafana/grafana/pkg/services/secrets/manager" migrator2 "github.com/grafana/grafana/pkg/services/secrets/migrator" "github.com/grafana/grafana/pkg/services/serviceaccounts" @@ -255,13 +255,14 @@ import ( "github.com/grafana/grafana/pkg/services/user/userimpl" "github.com/grafana/grafana/pkg/services/validations" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/storage/legacysql" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" database4 "github.com/grafana/grafana/pkg/storage/secret/database" "github.com/grafana/grafana/pkg/storage/secret/encryption" "github.com/grafana/grafana/pkg/storage/secret/metadata" "github.com/grafana/grafana/pkg/storage/secret/migrator" "github.com/grafana/grafana/pkg/storage/unified" - migrations3 "github.com/grafana/grafana/pkg/storage/unified/migrations" + migrations2 "github.com/grafana/grafana/pkg/storage/unified/migrations" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/storage/unified/sql" @@ -526,7 +527,15 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } - dualwriteService, err := dualwrite.ProvideService(featureToggles, kvStore, cfg) + legacyDatabaseProvider := legacysql.NewDatabaseProvider(sqlStore) + stubProvisioningService, err := provisioning.ProvideStubProvisioningService(cfg) + if err != nil { + return nil, err + } + migrationDashboardAccessor := legacy.ProvideMigratorDashboardAccessor(legacyDatabaseProvider, stubProvisioningService, accessControl, featureToggles) + unifiedMigrator := migrations2.ProvideUnifiedMigrator(migrationDashboardAccessor, resourceClient) + unifiedStorageMigrationService := migrations2.ProvideUnifiedStorageMigrationService(unifiedMigrator, cfg, sqlStore, kvStore) + dualwriteService, err := dualwrite.ProvideService(featureToggles, kvStore, cfg, unifiedStorageMigrationService) if err != nil { return nil, err } @@ -716,8 +725,8 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api csrfCSRF := csrf.ProvideCSRFFilter(cfg) playlistService := playlistimpl.ProvideService(sqlStore, tracingService) secretsMigrator := migrator2.ProvideSecretsMigrator(serviceService, secretsService, sqlStore, ossImpl, featureToggles) - dataSourceSecretMigrationService := migrations2.ProvideDataSourceMigrationService(service15, kvStore, featureToggles) - secretMigrationProviderImpl := migrations2.ProvideSecretMigrationProvider(serverLockService, dataSourceSecretMigrationService) + dataSourceSecretMigrationService := migrations3.ProvideDataSourceMigrationService(service15, kvStore, featureToggles) + secretMigrationProviderImpl := migrations3.ProvideSecretMigrationProvider(serverLockService, dataSourceSecretMigrationService) publicDashboardServiceImpl := service3.ProvideService(cfg, featureToggles, publicDashboardStoreImpl, queryServiceImpl, repositoryImpl, accessControl, publicDashboardServiceWrapperImpl, dashboardService, ossLicensingService) middleware := api2.ProvideMiddleware() apiApi := api2.ProvideApi(publicDashboardServiceImpl, routeRegisterImpl, accessControl, featureToggles, middleware, cfg, ossLicensingService) @@ -838,8 +847,6 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api dashboardUpdater := service8.ProvideDashboardUpdater(inProcBus, pluginstoreService, service14, importDashboardService, service13, pluginService, dashboardService) worker := garbagecollectionworker.ProvideWorker(cfg, secureValueMetadataStorage, keeperMetadataStorage, ossKeeperService) fixedRolesLoader := accesscontrol.ProvideFixedRolesLoader(acimplService, featureToggles) - legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) - unifiedStorageMigrationProviderImpl := migrations3.ProvideUnifiedStorageMigrationProvider(legacyMigrator, cfg, resourceClient, sqlStore) healthService, err := grpcserver.ProvideHealthService(cfg, grpcserverProvider) if err != nil { return nil, err @@ -892,7 +899,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } - provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, legacyMigrator, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) + provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) if err != nil { return nil, err } @@ -920,7 +927,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api } ossUserProtectionImpl := authinfoimpl.ProvideOSSUserProtectionService() registration := authnimpl.ProvideRegistration(cfg, authnService, orgService, userAuthTokenService, acimplService, permissionRegistry, apikeyService, userService, authService, ossUserProtectionImpl, loginattemptimplService, quotaService, authinfoimplService, renderingService, featureToggles, oauthtokenService, socialService, remoteCache, ldapImpl, ossImpl, tracingService, tempuserService, notificationService) - backgroundServiceRegistry := backgroundsvcs.ProvideBackgroundServiceRegistry(httpServer, alertNG, cleanUpService, grafanaLive, gateway, notificationService, pluginstoreService, renderingService, userAuthTokenService, tracingService, provisioningServiceImpl, usageStats, statscollectorService, grafanaService, pluginsService, internalMetricsService, secretsService, remoteCache, storageService, searchService, entityEventsService, serviceAccountsService, grpcserverProvider, secretMigrationProviderImpl, loginattemptimplService, supportbundlesimplService, metricService, keyRetriever, angulardetectorsproviderDynamic, apiserverService, anonDeviceService, ssosettingsimplService, pluginexternalService, plugininstallerService, zanzanaReconciler, appregistryService, dashboardUpdater, dashboardServiceImpl, worker, fixedRolesLoader, unifiedStorageMigrationProviderImpl, serviceImpl, serviceAccountsProxy, healthService, reflectionService, apiService, apiregistryService, idimplService, teamAPI, ssosettingsimplService, cloudmigrationService, registration) + backgroundServiceRegistry := backgroundsvcs.ProvideBackgroundServiceRegistry(httpServer, alertNG, cleanUpService, grafanaLive, gateway, notificationService, pluginstoreService, renderingService, userAuthTokenService, tracingService, provisioningServiceImpl, usageStats, statscollectorService, grafanaService, pluginsService, internalMetricsService, secretsService, remoteCache, storageService, searchService, entityEventsService, serviceAccountsService, grpcserverProvider, secretMigrationProviderImpl, loginattemptimplService, supportbundlesimplService, metricService, keyRetriever, angulardetectorsproviderDynamic, apiserverService, anonDeviceService, ssosettingsimplService, pluginexternalService, plugininstallerService, zanzanaReconciler, appregistryService, dashboardUpdater, dashboardServiceImpl, worker, fixedRolesLoader, serviceImpl, serviceAccountsProxy, healthService, reflectionService, apiService, apiregistryService, idimplService, teamAPI, ssosettingsimplService, cloudmigrationService, registration) usageStatsProvidersRegistry := usagestatssvcs.ProvideUsageStatsProvidersRegistry(acimplService, userService) server, err := New(opts, cfg, httpServer, acimplService, provisioningServiceImpl, backgroundServiceRegistry, usageStatsProvidersRegistry, statscollectorService, tracingService, featureToggles, registerer) if err != nil { @@ -1167,7 +1174,15 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - dualwriteService, err := dualwrite.ProvideService(featureToggles, kvStore, cfg) + legacyDatabaseProvider := legacysql.NewDatabaseProvider(sqlStore) + stubProvisioningService, err := provisioning.ProvideStubProvisioningService(cfg) + if err != nil { + return nil, err + } + migrationDashboardAccessor := legacy.ProvideMigratorDashboardAccessor(legacyDatabaseProvider, stubProvisioningService, accessControl, featureToggles) + unifiedMigrator := migrations2.ProvideUnifiedMigrator(migrationDashboardAccessor, resourceClient) + unifiedStorageMigrationService := migrations2.ProvideUnifiedStorageMigrationService(unifiedMigrator, cfg, sqlStore, kvStore) + dualwriteService, err := dualwrite.ProvideService(featureToggles, kvStore, cfg, unifiedStorageMigrationService) if err != nil { return nil, err } @@ -1359,8 +1374,8 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac csrfCSRF := csrf.ProvideCSRFFilter(cfg) playlistService := playlistimpl.ProvideService(sqlStore, tracingService) secretsMigrator := migrator2.ProvideSecretsMigrator(serviceService, secretsService, sqlStore, ossImpl, featureToggles) - dataSourceSecretMigrationService := migrations2.ProvideDataSourceMigrationService(service15, kvStore, featureToggles) - secretMigrationProviderImpl := migrations2.ProvideSecretMigrationProvider(serverLockService, dataSourceSecretMigrationService) + dataSourceSecretMigrationService := migrations3.ProvideDataSourceMigrationService(service15, kvStore, featureToggles) + secretMigrationProviderImpl := migrations3.ProvideSecretMigrationProvider(serverLockService, dataSourceSecretMigrationService) publicDashboardServiceImpl := service3.ProvideService(cfg, featureToggles, publicDashboardStoreImpl, queryServiceImpl, repositoryImpl, accessControl, publicDashboardServiceWrapperImpl, dashboardService, ossLicensingService) middleware := api2.ProvideMiddleware() apiApi := api2.ProvideApi(publicDashboardServiceImpl, routeRegisterImpl, accessControl, featureToggles, middleware, cfg, ossLicensingService) @@ -1481,8 +1496,6 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac dashboardUpdater := service8.ProvideDashboardUpdater(inProcBus, pluginstoreService, service14, importDashboardService, service13, pluginService, dashboardService) worker := garbagecollectionworker.ProvideWorker(cfg, secureValueMetadataStorage, keeperMetadataStorage, ossKeeperService) fixedRolesLoader := accesscontrol.ProvideFixedRolesLoader(acimplService, featureToggles) - legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) - unifiedStorageMigrationProviderImpl := migrations3.ProvideUnifiedStorageMigrationProvider(legacyMigrator, cfg, resourceClient, sqlStore) healthService, err := grpcserver.ProvideHealthService(cfg, grpcserverProvider) if err != nil { return nil, err @@ -1535,7 +1548,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, legacyMigrator, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) + provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) if err != nil { return nil, err } @@ -1563,7 +1576,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac } ossUserProtectionImpl := authinfoimpl.ProvideOSSUserProtectionService() registration := authnimpl.ProvideRegistration(cfg, authnService, orgService, userAuthTokenService, acimplService, permissionRegistry, apikeyService, userService, authService, ossUserProtectionImpl, loginattemptimplService, quotaService, authinfoimplService, renderingService, featureToggles, oauthtokentestService, socialService, remoteCache, ldapImpl, ossImpl, tracingService, tempuserService, notificationServiceMock) - backgroundServiceRegistry := backgroundsvcs.ProvideBackgroundServiceRegistry(httpServer, alertNG, cleanUpService, grafanaLive, gateway, notificationService, pluginstoreService, renderingService, userAuthTokenService, tracingService, provisioningServiceImpl, usageStats, statscollectorService, grafanaService, pluginsService, internalMetricsService, secretsService, remoteCache, storageService, searchService, entityEventsService, serviceAccountsService, grpcserverProvider, secretMigrationProviderImpl, loginattemptimplService, supportbundlesimplService, metricService, keyRetriever, angulardetectorsproviderDynamic, apiserverService, anonDeviceService, ssosettingsimplService, pluginexternalService, plugininstallerService, zanzanaReconciler, appregistryService, dashboardUpdater, dashboardServiceImpl, worker, fixedRolesLoader, unifiedStorageMigrationProviderImpl, serviceImpl, serviceAccountsProxy, healthService, reflectionService, apiService, apiregistryService, idimplService, teamAPI, ssosettingsimplService, cloudmigrationService, registration) + backgroundServiceRegistry := backgroundsvcs.ProvideBackgroundServiceRegistry(httpServer, alertNG, cleanUpService, grafanaLive, gateway, notificationService, pluginstoreService, renderingService, userAuthTokenService, tracingService, provisioningServiceImpl, usageStats, statscollectorService, grafanaService, pluginsService, internalMetricsService, secretsService, remoteCache, storageService, searchService, entityEventsService, serviceAccountsService, grpcserverProvider, secretMigrationProviderImpl, loginattemptimplService, supportbundlesimplService, metricService, keyRetriever, angulardetectorsproviderDynamic, apiserverService, anonDeviceService, ssosettingsimplService, pluginexternalService, plugininstallerService, zanzanaReconciler, appregistryService, dashboardUpdater, dashboardServiceImpl, worker, fixedRolesLoader, serviceImpl, serviceAccountsProxy, healthService, reflectionService, apiService, apiregistryService, idimplService, teamAPI, ssosettingsimplService, cloudmigrationService, registration) usageStatsProvidersRegistry := usagestatssvcs.ProvideUsageStatsProvidersRegistry(acimplService, userService) server, err := New(opts, cfg, httpServer, acimplService, provisioningServiceImpl, backgroundServiceRegistry, usageStatsProvidersRegistry, statscollectorService, tracingService, featureToggles, registerer) if err != nil { @@ -1759,7 +1772,7 @@ var withOTelSet = wire.NewSet( otelTracer, grpcserver.ProvideService, interceptors.ProvideAuthenticator, ) -var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatemanager.ProvideGrafanaService, updatemanager.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator3.ProvideService, legacy.ProvideLegacyMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, wire.Bind(new(installsync.ServerLock), new(*serverlock.ServerLockService)), annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, pushhttp.ProvideService, contexthandler.ProvideService, service12.ProvideService, wire.Bind(new(service12.LDAP), new(*service12.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, github.ProvideFactory, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), withOTelSet, testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service9.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service9.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption2.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), garbagecollectionworker.ProvideWorker, grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database5.DashboardSnapshotStore)), database5.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service10.ServiceImpl)), service10.ProvideService, service9.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service9.Service)), service9.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager3.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), dsquerierclient.NewNullQSDatasourceClientBuilder, expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, service7.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service7.DashboardServiceImpl)), service7.ProvideDashboardService, service7.ProvideDashboardProvisioningService, service7.ProvideDashboardPluginService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), wire.Bind(new(folder.LegacyService), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), service11.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service11.ImportDashboardService)), service8.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service8.Service)), service8.ProvideDashboardUpdater, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations2.ProvideDataSourceMigrationService, migrations2.ProvideSecretMigrationProvider, wire.Bind(new(migrations2.SecretMigrationProvider), new(*migrations2.SecretMigrationProviderImpl)), promtypemigration.ProvideAzurePromMigrationService, promtypemigration.ProvideAmazonPromMigrationService, promtypemigration.ProvidePromTypeMigrationProvider, wire.Bind(new(promtypemigration.PromTypeMigrationProvider), new(*promtypemigration.PromTypeMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, accesscontrol.ProvideFixedRolesLoader, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, caching.ProvideCachingServiceClient, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, metadata.ProvideSecureValueMetadataStorage, metadata.ProvideKeeperMetadataStorage, metadata.ProvideDecryptStorage, decrypt.ProvideDecryptAuthorizer, wire.Value([]decrypt.ExtraOwnerDecrypter(nil)), decrypt.ProvideDecryptService, inline.ProvideInlineSecureValueService, encryption.ProvideDataKeyStorage, encryption.ProvideGlobalDataKeyStorage, encryption.ProvideEncryptedValueStorage, encryption.ProvideGlobalEncryptedValueStorage, encryption.ProvideEncryptedValueMigrationExecutor, service5.ProvideSecureValueService, validator.ProvideKeeperValidator, validator.ProvideSecureValueValidator, mutator.ProvideKeeperMutator, mutator.ProvideSecureValueMutator, migrator.NewWithEngine, database4.ProvideDatabase, clock.ProvideClock, wire.Bind(new(contracts.Database), new(*database4.Database)), wire.Bind(new(contracts.Clock), new(*clock.Clock)), manager2.ProvideEncryptionManager, service4.ProvideAESGCMCipherService, resource.ProvideStorageMetrics, resource.ProvideIndexMetrics, migrations3.ProvideUnifiedStorageMigrationProvider, wire.Bind(new(migrations3.UnifiedStorageMigrationProvider), new(*migrations3.UnifiedStorageMigrationProviderImpl)), apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet, client.ProvideK8sClientWithFallback) +var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatemanager.ProvideGrafanaService, updatemanager.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator3.ProvideService, provisioning.ProvideStubProvisioningService, legacy.ProvideMigratorDashboardAccessor, migrations2.ProvideUnifiedMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, wire.Bind(new(installsync.ServerLock), new(*serverlock.ServerLockService)), annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, pushhttp.ProvideService, contexthandler.ProvideService, service12.ProvideService, wire.Bind(new(service12.LDAP), new(*service12.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, github.ProvideFactory, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), withOTelSet, testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service9.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service9.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption2.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), garbagecollectionworker.ProvideWorker, grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database5.DashboardSnapshotStore)), database5.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service10.ServiceImpl)), service10.ProvideService, service9.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service9.Service)), service9.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager3.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), dsquerierclient.NewNullQSDatasourceClientBuilder, expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, service7.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service7.DashboardServiceImpl)), service7.ProvideDashboardService, service7.ProvideDashboardProvisioningService, service7.ProvideDashboardPluginService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), wire.Bind(new(folder.LegacyService), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), service11.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service11.ImportDashboardService)), service8.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service8.Service)), service8.ProvideDashboardUpdater, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations3.ProvideDataSourceMigrationService, migrations3.ProvideSecretMigrationProvider, wire.Bind(new(migrations3.SecretMigrationProvider), new(*migrations3.SecretMigrationProviderImpl)), promtypemigration.ProvideAzurePromMigrationService, promtypemigration.ProvideAmazonPromMigrationService, promtypemigration.ProvidePromTypeMigrationProvider, wire.Bind(new(promtypemigration.PromTypeMigrationProvider), new(*promtypemigration.PromTypeMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, accesscontrol.ProvideFixedRolesLoader, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, caching.ProvideCachingServiceClient, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, metadata.ProvideSecureValueMetadataStorage, metadata.ProvideKeeperMetadataStorage, metadata.ProvideDecryptStorage, decrypt.ProvideDecryptAuthorizer, wire.Value([]decrypt.ExtraOwnerDecrypter(nil)), decrypt.ProvideDecryptService, inline.ProvideInlineSecureValueService, encryption.ProvideDataKeyStorage, encryption.ProvideGlobalDataKeyStorage, encryption.ProvideEncryptedValueStorage, encryption.ProvideGlobalEncryptedValueStorage, encryption.ProvideEncryptedValueMigrationExecutor, service5.ProvideSecureValueService, validator.ProvideKeeperValidator, validator.ProvideSecureValueValidator, mutator.ProvideKeeperMutator, mutator.ProvideSecureValueMutator, migrator.NewWithEngine, database4.ProvideDatabase, clock.ProvideClock, wire.Bind(new(contracts.Database), new(*database4.Database)), wire.Bind(new(contracts.Clock), new(*clock.Clock)), manager2.ProvideEncryptionManager, service4.ProvideAESGCMCipherService, resource.ProvideStorageMetrics, resource.ProvideIndexMetrics, migrations2.ProvideUnifiedStorageMigrationService, apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet, client.ProvideK8sClientWithFallback) var wireSet = wire.NewSet( wireBasicSet, metrics.WireSet, sqlstore.ProvideService, metrics2.ProvideService, wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), wire.Bind(new(notifications.WebhookSender), new(*notifications.NotificationService)), wire.Bind(new(notifications.EmailSender), new(*notifications.NotificationService)), wire.Bind(new(db.DB), new(*sqlstore.SQLStore)), prefimpl.ProvideService, oauthtoken.ProvideService, wire.Bind(new(oauthtoken.OAuthTokenService), new(*oauthtoken.Service)), wire.Bind(new(cleanup.AlertRuleService), new(*store2.DBstore)), diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index b2b134ee4a9..9f069f807fc 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -62,6 +62,7 @@ import ( "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/services/validations" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/storage/legacysql" "github.com/grafana/grafana/pkg/storage/unified" "github.com/grafana/grafana/pkg/storage/unified/resource" search2 "github.com/grafana/grafana/pkg/storage/unified/search" @@ -98,6 +99,7 @@ var wireExtsBasicSet = wire.NewSet( wire.Bind(new(validations.DataSourceRequestURLValidator), new(*validations.OSSDataSourceRequestURLValidator)), provisioning.ProvideService, wire.Bind(new(provisioning.ProvisioningService), new(*provisioning.ProvisioningServiceImpl)), + legacysql.NewDatabaseProvider, backgroundsvcs.ProvideBackgroundServiceRegistry, wire.Bind(new(registry.BackgroundServiceRegistry), new(*backgroundsvcs.BackgroundServiceRegistry)), migrations.ProvideOSSMigrations, diff --git a/pkg/services/apiserver/appinstaller/storage.go b/pkg/services/apiserver/appinstaller/storage.go index 3ecc34cce58..98989508858 100644 --- a/pkg/services/apiserver/appinstaller/storage.go +++ b/pkg/services/apiserver/appinstaller/storage.go @@ -107,7 +107,7 @@ func NewDualWriter( if currentMode != mode { klog.Warningf("Requested DualWrite mode: %d, but using %d for %+v", mode, currentMode, gr) } - return dualwrite.NewDualWriter(gr, currentMode, legacy, storage) + return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage) } func getRequestInfo(gr schema.GroupResource, namespaceMapper request.NamespaceMapper) *k8srequest.RequestInfo { diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index 98a9c08ed25..603c49f3cb2 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -379,7 +379,7 @@ func InstallAPIs( case grafanarest.Mode4, grafanarest.Mode5: return storage, nil default: - return dualwrite.NewDualWriter(gr, currentMode, legacy, storage) + return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage) } } } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 7d4ee3c0d5a..28da416536c 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -97,7 +97,7 @@ type service struct { authorizer *authorizer.GrafanaAuthorizer serverLockService builder.ServerLockService - storageStatus dualwrite.Service + dualWriter dualwrite.Service kvStore kvstore.KVStore pluginClient plugins.Client @@ -127,7 +127,7 @@ func ProvideService( datasources datasource.ScopedPluginDatasourceProvider, contextProvider datasource.PluginContextWrapper, pluginStore pluginstore.Store, - storageStatus dualwrite.Service, + dualWriter dualwrite.Service, unified resource.ResourceClient, secrets secret.InlineSecureValueSupport, restConfigProvider RestConfigProvider, @@ -158,7 +158,7 @@ func ProvideService( contextProvider: contextProvider, pluginStore: pluginStore, serverLockService: serverLockService, - storageStatus: storageStatus, + dualWriter: dualWriter, unified: unified, secrets: secrets, restConfigProvider: restConfigProvider, @@ -385,12 +385,17 @@ func (s *service) start(ctx context.Context) error { } // Install the API group+version for existing builders - err = builder.InstallAPIs(s.scheme, s.codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions, + err = builder.InstallAPIs(s.scheme, + s.codecs, + server, + serverConfig.RESTOptionsGetter, + builders, + o.StorageOptions, s.metrics, request.GetNamespaceMapper(s.cfg), kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService, - s.storageStatus, + s.dualWriter, optsregister, s.features, s.dualWriterMetrics, @@ -409,7 +414,7 @@ func (s *service) start(ctx context.Context) error { kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService, request.GetNamespaceMapper(s.cfg), - s.storageStatus, + s.dualWriter, s.dualWriterMetrics, s.builderMetrics, serverConfig.MergedResourceConfig, diff --git a/pkg/services/provisioning/stubs.go b/pkg/services/provisioning/stubs.go new file mode 100644 index 00000000000..b40e66aac5c --- /dev/null +++ b/pkg/services/provisioning/stubs.go @@ -0,0 +1,69 @@ +package provisioning + +import ( + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/provisioning/dashboards" + "github.com/grafana/grafana/pkg/setting" +) + +type StubProvisioningService interface { + GetDashboardProvisionerResolvedPath(name string) string + GetAllowUIUpdatesFromConfig(name string) bool +} + +func ProvideStubProvisioningService(cfg *setting.Cfg) (StubProvisioningService, error) { + return NewStubProvisioning(cfg.ProvisioningPath) +} + +func NewStubProvisioning(path string) (StubProvisioningService, error) { + cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards")) + if err != nil { + return nil, err + } + stub := &stubProvisioning{ + path: make(map[string]string), + allowUIUpdates: make(map[string]bool), + log: log.New("provisioning.stub"), + } + for _, cfg := range cfgs { + stub.path[cfg.Name] = cfg.Options["path"].(string) + stub.allowUIUpdates[cfg.Name] = cfg.AllowUIUpdates + } + return stub, nil +} + +type stubProvisioning struct { + path map[string]string // name > options.path + allowUIUpdates map[string]bool + log log.Logger +} + +func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool { + return s.allowUIUpdates[name] +} + +func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string { + path := s.path[name] + if _, err := os.Stat(path); os.IsNotExist(err) { + s.log.Warn("Cannot read directory", "error", err) + } + + path, err := filepath.Abs(path) + if err != nil { + s.log.Warn("Could not create absolute path", "path", path, "error", err) + } + + path, err = filepath.EvalSymlinks(path) + if err != nil { + s.log.Warn("Failed to read content of symlinked path", "path", path, "error", err) + } + + if path == "" { + path = s.path[name] + s.log.Info("falling back to original path due to EvalSymlink/Abs failure") + } + return path +} diff --git a/pkg/storage/legacysql/dualwrite/dualwriter_mode1_test.go b/pkg/storage/legacysql/dualwrite/dualwriter_mode1_test.go index b864e3a360d..777d791d4e5 100644 --- a/pkg/storage/legacysql/dualwrite/dualwriter_mode1_test.go +++ b/pkg/storage/legacysql/dualwrite/dualwriter_mode1_test.go @@ -71,7 +71,7 @@ func TestMode1_Create(t *testing.T) { tt.setupStorageFn(us.Mock) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{}) @@ -154,7 +154,7 @@ func TestMode1_Get(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{}) @@ -217,7 +217,7 @@ func TestMode1_List(t *testing.T) { tt.setupStorageFn(us.Mock) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) _, err = dw.List(context.Background(), &metainternalversion.ListOptions{}) @@ -286,7 +286,7 @@ func TestMode1_Delete(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) obj, _, err := dw.Delete(context.Background(), name, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{}) @@ -361,7 +361,7 @@ func TestMode1_DeleteCollection(t *testing.T) { tt.setupStorageFn(us.Mock, tt.input) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{}) @@ -434,7 +434,7 @@ func TestMode1_Update(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode1, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode1, ls, us) require.NoError(t, err) obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{}) diff --git a/pkg/storage/legacysql/dualwrite/dualwriter_mode2_test.go b/pkg/storage/legacysql/dualwrite/dualwriter_mode2_test.go index cd1e87b0b9a..a45dfa1285d 100644 --- a/pkg/storage/legacysql/dualwrite/dualwriter_mode2_test.go +++ b/pkg/storage/legacysql/dualwrite/dualwriter_mode2_test.go @@ -69,7 +69,7 @@ func TestMode2_Create(t *testing.T) { tt.setupStorageFn(us.Mock, tt.input) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{}) @@ -154,7 +154,7 @@ func TestMode2_Get(t *testing.T) { tt.setupStorageFn(us.Mock, tt.input) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{}) @@ -229,7 +229,7 @@ func TestMode2_List(t *testing.T) { tt.setupStorageFn(us.Mock) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{}) @@ -331,7 +331,7 @@ func TestMode2_Delete(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{}) @@ -401,7 +401,7 @@ func TestMode2_DeleteCollection(t *testing.T) { tt.setupStorageFn(us.Mock) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{}) @@ -471,7 +471,7 @@ func TestMode2_Update(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode2, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode2, ls, us) require.NoError(t, err) obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{}) diff --git a/pkg/storage/legacysql/dualwrite/dualwriter_mode3_test.go b/pkg/storage/legacysql/dualwrite/dualwriter_mode3_test.go index 3076d8a9409..d2485efb762 100644 --- a/pkg/storage/legacysql/dualwrite/dualwriter_mode3_test.go +++ b/pkg/storage/legacysql/dualwrite/dualwriter_mode3_test.go @@ -75,7 +75,7 @@ func TestMode3_Create(t *testing.T) { tt.setupStorageFn(us.Mock, tt.input) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{}) @@ -134,7 +134,7 @@ func TestMode3_Get(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{}) @@ -185,7 +185,7 @@ func TestMode3_List(t *testing.T) { tt.setupStorageFn(us.Mock, &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}}) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) res, err := dw.List(context.Background(), &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}}) @@ -276,7 +276,7 @@ func TestMode3_Delete(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{}) @@ -346,7 +346,7 @@ func TestMode3_DeleteCollection(t *testing.T) { tt.setupStorageFn(us.Mock) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{}) @@ -416,7 +416,7 @@ func TestMode3_Update(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - dw, err := NewDualWriter(kind, rest.Mode3, ls, us) + dw, err := NewStaticStorage(kind, rest.Mode3, ls, us) require.NoError(t, err) obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{}) diff --git a/pkg/storage/legacysql/dualwrite/runtime.go b/pkg/storage/legacysql/dualwrite/runtime.go index 180beeb255a..b9c5355050c 100644 --- a/pkg/storage/legacysql/dualwrite/runtime.go +++ b/pkg/storage/legacysql/dualwrite/runtime.go @@ -14,37 +14,6 @@ import ( grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" ) -func (m *service) NewStorage(gr schema.GroupResource, legacy grafanarest.Storage, unified grafanarest.Storage) (grafanarest.Storage, error) { - status, err := m.Status(context.Background(), gr) - if err != nil { - return nil, err - } - - if m.enabled && status.Runtime { - // Dynamic storage behavior - return &runtimeDualWriter{ - service: m, - legacy: legacy, - unified: unified, - dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read - gr: gr, - }, nil - } - - if status.ReadUnified { - if status.WriteLegacy { - // Write both, read unified - return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil - } - return unified, nil - } - if status.WriteUnified { - // Write both, read legacy - return &dualWriter{legacy: legacy, unified: unified}, nil - } - return legacy, nil -} - // The runtime dual writer implements the various modes we have described as: mode:1/2/3/4/5 // However the behavior can be configured at runtime rather than just at startup. // When a resource is marked as "migrating", all write requests will be 503 unavailable diff --git a/pkg/storage/legacysql/dualwrite/runtime_test.go b/pkg/storage/legacysql/dualwrite/runtime_test.go index b59801e3da8..01dd79647a7 100644 --- a/pkg/storage/legacysql/dualwrite/runtime_test.go +++ b/pkg/storage/legacysql/dualwrite/runtime_test.go @@ -77,7 +77,7 @@ func TestRuntime_Create(t *testing.T) { tt.setupStorageFn(us.Mock, tt.input) } - m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil) + m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator()) require.NoError(t, err) dw, err := m.NewStorage(kind, ls, us) require.NoError(t, err) @@ -150,7 +150,7 @@ func TestRuntime_Get(t *testing.T) { tt.setupStorageFn(us.Mock, name) } - m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil) + m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator()) require.NoError(t, err) dw, err := m.NewStorage(kind, ls, us) require.NoError(t, err) @@ -235,7 +235,7 @@ func TestRuntime_CreateWhileMigrating(t *testing.T) { } // Shared provider across all tests - dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil) + dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator()) require.NoError(t, err) for _, tt := range tests { diff --git a/pkg/storage/legacysql/dualwrite/service.go b/pkg/storage/legacysql/dualwrite/service.go index 77ecc761de1..8ac88deb281 100644 --- a/pkg/storage/legacysql/dualwrite/service.go +++ b/pkg/storage/legacysql/dualwrite/service.go @@ -12,8 +12,28 @@ import ( "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" + unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations/contract" ) +// fakeMigrator is a no-op implementation of UnifiedStorageMigrationService +type fakeMigrator struct{} + +func (f *fakeMigrator) Run(ctx context.Context) error { + return nil +} + +var _ unifiedmigrations.UnifiedStorageMigrationService = (*fakeMigrator)(nil) + +func NewFakeMigrator() unifiedmigrations.UnifiedStorageMigrationService { + return &fakeMigrator{} +} + +func NewFakeConfig() *setting.Cfg { + return &setting.Cfg{ + UnifiedStorage: make(map[string]setting.UnifiedStorageConfig), + } +} + func ProvideStaticServiceForTests(cfg *setting.Cfg) Service { if cfg == nil { cfg = &setting.Cfg{} @@ -25,7 +45,14 @@ func ProvideService( features featuremgmt.FeatureToggles, kv kvstore.KVStore, cfg *setting.Cfg, + migrator unifiedmigrations.UnifiedStorageMigrationService, ) (Service, error) { + // Ensure migrations have run before starting dualwrite + err := migrator.Run(context.Background()) + if err != nil { + return nil, fmt.Errorf("unable to start dualwrite service due to migration error: %w", err) + } + //nolint:staticcheck // not yet migrated to OpenFeature enabled := features.IsEnabledGlobally(featuremgmt.FlagManagedDualWriter) || features.IsEnabledGlobally(featuremgmt.FlagProvisioning) // required for git provisioning @@ -64,6 +91,37 @@ type service struct { enabled bool } +func (m *service) NewStorage(gr schema.GroupResource, legacy rest.Storage, unified rest.Storage) (rest.Storage, error) { + status, err := m.Status(context.Background(), gr) + if err != nil { + return nil, err + } + + if m.enabled && status.Runtime { + // Dynamic storage behavior + return &runtimeDualWriter{ + service: m, + legacy: legacy, + unified: unified, + dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read + gr: gr, + }, nil + } + + if status.ReadUnified { + if status.WriteLegacy { + // Write both, read unified + return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil + } + return unified, nil + } + if status.WriteUnified { + // Write both, read legacy + return &dualWriter{legacy: legacy, unified: unified}, nil + } + return legacy, nil +} + // Hardcoded list of resources that should be controlled by the database (eventually everything?) func (m *service) ShouldManage(gr schema.GroupResource) bool { if !m.enabled { diff --git a/pkg/storage/legacysql/dualwrite/service_test.go b/pkg/storage/legacysql/dualwrite/service_test.go index a083e6470be..f9fcb9f9b38 100644 --- a/pkg/storage/legacysql/dualwrite/service_test.go +++ b/pkg/storage/legacysql/dualwrite/service_test.go @@ -17,7 +17,7 @@ import ( func TestService(t *testing.T) { t.Run("dynamic", func(t *testing.T) { ctx := context.Background() - mode, err := ProvideService(featuremgmt.WithFeatures(), kvstore.NewFakeKVStore(), nil) + mode, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagProvisioning), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator()) require.NoError(t, err) gr := schema.GroupResource{Group: "ggg", Resource: "rrr"} @@ -122,7 +122,7 @@ func TestService(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() - svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg) + svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg, NewFakeMigrator()) if tc.error != "" { require.ErrorContains(t, err, tc.error) require.Nil(t, svc, "expect a nil service when an error exts") diff --git a/pkg/storage/legacysql/dualwrite/static.go b/pkg/storage/legacysql/dualwrite/static.go index 3835cf80752..bd51b9b6ac6 100644 --- a/pkg/storage/legacysql/dualwrite/static.go +++ b/pkg/storage/legacysql/dualwrite/static.go @@ -10,8 +10,8 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -// NewDualWriter -- temporary shim -func NewDualWriter( +// NewStaticStorage -- temporary shim +func NewStaticStorage( gr schema.GroupResource, mode rest.DualWriterMode, legacy rest.Storage, diff --git a/pkg/storage/unified/migrations/contract/migrations.go b/pkg/storage/unified/migrations/contract/migrations.go new file mode 100644 index 00000000000..fe1c42cea21 --- /dev/null +++ b/pkg/storage/unified/migrations/contract/migrations.go @@ -0,0 +1,12 @@ +package contract + +import ( + "github.com/grafana/grafana/pkg/registry" +) + +// UnifiedStorageMigrationService provides unified storage migrations as a background service. +// This interface is defined in a separate package to avoid import cycles between +// the migrations implementation and packages that need to depend on it (like dualwrite). +type UnifiedStorageMigrationService interface { + registry.BackgroundService +} diff --git a/pkg/storage/unified/migrations/dashboard_folder_migration.go b/pkg/storage/unified/migrations/dashboard_folder_migration.go deleted file mode 100644 index 96c5a824c8a..00000000000 --- a/pkg/storage/unified/migrations/dashboard_folder_migration.go +++ /dev/null @@ -1,93 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/grafana/authlib/types" - "github.com/grafana/grafana/pkg/apimachinery/identity" - "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/storage/unified/resource" - "github.com/grafana/grafana/pkg/util/xorm" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -const ( - FoldersAndDashboardsMigrationID = "folders and dashboards migration" - UnifiedStorageDataMigrationSQL = "unified storage data migration" -) - -type dashboardAndFolderMigration struct { - migrator.MigrationBase - legacyMigrator legacy.LegacyMigrator - bulkStoreClient resource.ResourceClient -} - -var _ migrator.CodeMigration = (*dashboardAndFolderMigration)(nil) - -// SQL implements migrator.Migration interface. Returns a description string. -func (sp *dashboardAndFolderMigration) SQL(dialect migrator.Dialect) string { - return UnifiedStorageDataMigrationSQL -} - -func (sp *dashboardAndFolderMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - ctx := context.Background() - logger := mg.Logger - - resources := []schema.GroupResource{ - { - Group: "folder.grafana.app", - Resource: "folders", - }, - { - Group: "dashboard.grafana.app", - Resource: "dashboards", - }, - } - - storageMigrator := newUnifiedStorageMigrator(sp.legacyMigrator, sp.bulkStoreClient, resources, "unified-storage-migration.folders-dashboards") - - orgs, err := sp.getAllOrgs(sess) - if err != nil { - logger.Error("failed to get organizations for folders and dashboards migration", "error", err) - return fmt.Errorf("failed to get organizations: %w", err) - } - - if len(orgs) == 0 { - logger.Info("No organizations found to migrate, skipping migration") - return nil - } - - logger.Info("Starting migration for all organizations", "org_count", len(orgs)) - - for _, org := range orgs { - namespace := types.OrgNamespaceFormatter(org.ID) - logger.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace) - - // Create a service identity context for this namespace to authenticate with unified storage - migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace) - - if err := storageMigrator.executeMigration(migrationCtx, sess, mg, namespace); err != nil { - logger.Error("migration failed for organization", "org_id", org.ID, "org_name", org.Name, "error", err) - return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err) - } - } - - logger.Info("Migration completed successfully for all organizations", "org_count", len(orgs)) - return nil -} - -type orgInfo struct { - ID int64 `xorm:"id"` - Name string `xorm:"name"` -} - -func (sp *dashboardAndFolderMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) { - var orgs []orgInfo - err := sess.Table("org").Cols("id", "name").Find(&orgs) - if err != nil { - return nil, err - } - return orgs, nil -} diff --git a/pkg/storage/unified/migrations/migrations.go b/pkg/storage/unified/migrations/migrations.go deleted file mode 100644 index a30777e308a..00000000000 --- a/pkg/storage/unified/migrations/migrations.go +++ /dev/null @@ -1,112 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "os" - - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/registry" - "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/storage/unified/resource" - "github.com/prometheus/client_golang/prometheus" - "go.opentelemetry.io/otel" -) - -var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations") -var logger = log.New("storage.unified.migrations") - -// UnifiedStorageMigrationProvider provides unified storage migrations as a background service -type UnifiedStorageMigrationProvider interface { - registry.BackgroundService -} - -type UnifiedStorageMigrationProviderImpl struct { - legacyMigrator legacy.LegacyMigrator - cfg *setting.Cfg - client resource.ResourceClient - sqlStore db.DB -} - -var _ UnifiedStorageMigrationProvider = (*UnifiedStorageMigrationProviderImpl)(nil) - -// ProvideUnifiedStorageMigrationProvider is a Wire provider that creates the migration service. -// The service implements registry.BackgroundService and runs migrations during server startup. -func ProvideUnifiedStorageMigrationProvider( - legacyMigrator legacy.LegacyMigrator, - cfg *setting.Cfg, - client resource.ResourceClient, - sqlStore db.DB, -) *UnifiedStorageMigrationProviderImpl { - return &UnifiedStorageMigrationProviderImpl{ - legacyMigrator: legacyMigrator, - cfg: cfg, - client: client, - sqlStore: sqlStore, - } -} - -// Run executes unified storage migrations as a background service. -// This blocks until migrations complete. If migrations fail, an error is returned -// which will prevent Grafana from starting. -func (p *UnifiedStorageMigrationProviderImpl) Run(ctx context.Context) error { - // skip migrations in test environments to prevent integration test timeouts. - if os.Getenv("GRAFANA_TEST_DB") != "" { - return nil - } - // skip migrations if disabled in config - if p.cfg.DisableDataMigrations { - logger.Info("Data migrations are disabled, skipping") - return nil - } - // TODO: Re-enable once migrations are ready - // return RegisterMigrations(p.legacyMigrator, p.cfg, p.client, p.sqlStore) - return nil -} - -// RegisterMigrations initializes and registers all unified storage migrations. -// This function is the entry point for all data migrations from legacy storage -// to unified storage. It returns an error if migrations fail, preventing Grafana -// from starting with inconsistent data. -func RegisterMigrations( - legacyMigrator legacy.LegacyMigrator, - cfg *setting.Cfg, - client resource.ResourceClient, - sqlStore db.DB, -) error { - ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations") - defer span.End() - mg := migrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unified_storage") - mg.AddCreateMigration() - - if err := prometheus.Register(mg); err != nil { - logger.Warn("Failed to register migrator metrics", "error", err) - } - - // Add new migration registrations here for each resource type - registerDashboardAndFolderMigration(mg, legacyMigrator, client) - - // Run all registered migrations (blocking) - sec := cfg.Raw.Section("database") - if err := mg.RunMigrations(ctx, sec.Key("migration_locking").MustBool(true), sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil { - return fmt.Errorf("unified storage data migration failed: %w", err) - } - - logger.Info("Unified storage migrations completed successfully") - return nil -} - -func registerDashboardAndFolderMigration( - mg *migrator.Migrator, - legacyMigrator legacy.LegacyMigrator, - bulkStoreClient resource.ResourceClient, -) { - migration := &dashboardAndFolderMigration{ - legacyMigrator: legacyMigrator, - bulkStoreClient: bulkStoreClient, - } - mg.AddMigration(FoldersAndDashboardsMigrationID, migration) -} diff --git a/pkg/storage/unified/migrations/migrator.go b/pkg/storage/unified/migrations/migrator.go index cab6ce38432..bdbb69f98a1 100644 --- a/pkg/storage/unified/migrations/migrator.go +++ b/pkg/storage/unified/migrations/migrator.go @@ -3,164 +3,200 @@ package migrations import ( "context" "fmt" - "time" - "github.com/grafana/authlib/types" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "google.golang.org/grpc/metadata" + + authlib "github.com/grafana/authlib/types" + + v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" + folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" - "github.com/grafana/grafana/pkg/util/xorm" - "k8s.io/apimachinery/pkg/runtime/schema" ) -// StorageMigrator defines the interface for executing unified storage migrations -type StorageMigrator interface { - executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error +// Read from legacy and write into unified storage +// +//go:generate mockery --name UnifiedMigrator --structname MockUnifiedMigrator --inpackage --filename migrator_mock.go --with-expecter +type UnifiedMigrator interface { + Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) } -type unifiedStorageMigrator struct { - migrator legacy.LegacyMigrator - bulkStoreClient resource.ResourceClient - resources []schema.GroupResource - log log.Logger +// unifiedMigration handles the migration of legacy resources to unified storage +type unifiedMigration struct { + legacy.MigrationDashboardAccessor + streamProvider streamProvider + log log.Logger } -func newUnifiedStorageMigrator(migrator legacy.LegacyMigrator, bulkStoreClient resource.ResourceClient, resources []schema.GroupResource, logPrefix string) StorageMigrator { - return &unifiedStorageMigrator{ - migrator: migrator, - bulkStoreClient: bulkStoreClient, - resources: resources, - log: log.New(logPrefix), - } +// streamProvider abstracts the different ways to create a bulk process stream +type streamProvider interface { + createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) } -func (m *unifiedStorageMigrator) executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error { - startTime := time.Now() - m.log.Info("Starting unified storage migration", "namespace", namespace, "resources", m.resources) - - opts := legacy.MigrateOptions{ - Namespace: namespace, - Store: m.bulkStoreClient, - LargeObjects: nil, // Not using large object support to avoid import cycles - Resources: m.resources, - WithHistory: true, // Migrate with full history - OnlyCount: false, - Progress: func(count int, msg string) { - m.log.Info("Migration progress", "count", count, "message", msg) - }, - } - - // Execute the migration via legacy migrator - response, err := m.migrator.Migrate(ctx, opts) - if err != nil { - m.log.Error("Migration failed", "error", err, "duration", time.Since(startTime)) - return fmt.Errorf("failed to migrate resources: %w", err) - } - - // Validate the migration results - if err := m.validateMigration(sess, response); err != nil { - m.log.Error("Migration validation failed", "error", err, "duration", time.Since(startTime)) - return fmt.Errorf("migration validation failed: %w", err) - } - - m.log.Info("Migration completed successfully", - "duration", time.Since(startTime), - "processed", response.Processed, - "summaries", len(response.Summary), - "rejected", len(response.Rejected)) - - return nil +// resourceClientStreamProvider creates streams using resource.ResourceClient +type resourceClientStreamProvider struct { + client resource.ResourceClient } -func (m *unifiedStorageMigrator) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error { - // Check for rejected items - if len(response.Rejected) > 0 { - m.log.Warn("Migration had rejected items", "count", len(response.Rejected)) - for i, rejected := range response.Rejected { - if i < 10 { // Log first 10 rejected items - m.log.Warn("Rejected item", - "namespace", rejected.Key.Namespace, - "group", rejected.Key.Group, - "resource", rejected.Key.Resource, - "name", rejected.Key.Name, - "reason", rejected.Error) - } +func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) { + // Build collection settings for resource client + settings := resource.BulkSettings{ + RebuildCollection: true, + SkipValidation: true, + } + for _, res := range opts.Resources { + switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { + case "folder.grafana.app/folders": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: folders.GROUP, + Resource: folders.RESOURCE, + }) + case "dashboard.grafana.app/librarypanels": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: v1beta1.GROUP, + Resource: v1beta1.LIBRARY_PANEL_RESOURCE, + }) + case "dashboard.grafana.app/dashboards": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: v1beta1.GROUP, + Resource: v1beta1.DASHBOARD_RESOURCE, + }) } - // Rejections are not fatal - they may be expected for invalid data + } + ctx = metadata.NewOutgoingContext(ctx, settings.ToMD()) + return r.client.BulkProcess(ctx) +} + +// bulkStoreClientStreamProvider creates streams using resourcepb.BulkStoreClient +type bulkStoreClientStreamProvider struct { + client resourcepb.BulkStoreClient +} + +func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) { + // Build collection settings for resource client + settings := resource.BulkSettings{ + RebuildCollection: true, + SkipValidation: true, + } + for _, res := range opts.Resources { + switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { + case "folder.grafana.app/folders": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: folders.GROUP, + Resource: folders.RESOURCE, + }) + case "dashboard.grafana.app/librarypanels": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: v1beta1.GROUP, + Resource: v1beta1.LIBRARY_PANEL_RESOURCE, + }) + case "dashboard.grafana.app/dashboards": + settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{ + Namespace: opts.Namespace, + Group: v1beta1.GROUP, + Resource: v1beta1.DASHBOARD_RESOURCE, + }) + } + } + ctx = metadata.NewOutgoingContext(ctx, settings.ToMD()) + return b.client.BulkProcess(ctx) +} + +// This can migrate Folders, Dashboards and LibraryPanels +func ProvideUnifiedMigrator( + dashboardAccess legacy.MigrationDashboardAccessor, + client resource.ResourceClient, +) UnifiedMigrator { + return newUnifiedMigrator( + dashboardAccess, + &resourceClientStreamProvider{client: client}, + log.New("storage.unified.migrator"), + ) +} + +func ProvideUnifiedMigratorParquet( + dashboardAccess legacy.MigrationDashboardAccessor, + client resourcepb.BulkStoreClient, +) UnifiedMigrator { + return newUnifiedMigrator( + dashboardAccess, + &bulkStoreClientStreamProvider{client: client}, + log.New("storage.unified.migrator.parquet"), + ) +} + +func newUnifiedMigrator( + dashboardAccess legacy.MigrationDashboardAccessor, + streamProvider streamProvider, + log log.Logger, +) UnifiedMigrator { + return &unifiedMigration{ + MigrationDashboardAccessor: dashboardAccess, + streamProvider: streamProvider, + log: log, + } +} + +// migrate function -- works for a single kind +type migratorFunc = func(ctx context.Context, orgId int64, opts legacy.MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*legacy.BlobStoreInfo, error) + +func (m *unifiedMigration) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) { + info, err := authlib.ParseNamespace(opts.Namespace) + if err != nil { + return nil, err + } + if opts.Progress == nil { + opts.Progress = func(count int, msg string) {} // noop } - // Validate counts for each resource type - for _, summary := range response.Summary { - legacyCount, err := m.getLegacyCount(sess, summary.Group, summary.Resource, summary.Namespace) + if len(opts.Resources) < 1 { + return nil, fmt.Errorf("missing resource selector") + } + + if opts.OnlyCount { + return m.CountResources(ctx, opts) + } + + stream, err := m.streamProvider.createStream(ctx, opts) + if err != nil { + return nil, err + } + + migratorFuncs := []migratorFunc{} + for _, res := range opts.Resources { + switch fmt.Sprintf("%s/%s", res.Group, res.Resource) { + case "folder.grafana.app/folders": + migratorFuncs = append(migratorFuncs, m.MigrateFolders) + case "dashboard.grafana.app/librarypanels": + migratorFuncs = append(migratorFuncs, m.MigrateLibraryPanels) + case "dashboard.grafana.app/dashboards": + migratorFuncs = append(migratorFuncs, m.MigrateDashboards) + default: + return nil, fmt.Errorf("unsupported resource: %s", res) + } + } + + // Execute migrations + blobStore := legacy.BlobStoreInfo{} + m.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID) + for _, fn := range migratorFuncs { + blobs, err := fn(ctx, info.OrgID, opts, stream) if err != nil { - return fmt.Errorf("failed to get legacy count for %s/%s: %w", summary.Group, summary.Resource, err) + m.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace) + return nil, err } - - // Account for rejected items in validation - expectedCount := summary.Count + int64(len(response.Rejected)) - - m.log.Info("Count validation", - "resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group), - "namespace", summary.Namespace, - "legacy_count", legacyCount, - "unified_count", summary.Count, - "rejected", len(response.Rejected), - "history", summary.History) - - // Validate that we migrated all items (allowing for rejected items) - if legacyCount > expectedCount { - return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d", - summary.Resource, summary.Group, summary.Namespace, - legacyCount, summary.Count, len(response.Rejected)) + if blobs != nil { + blobStore.Count += blobs.Count + blobStore.Size += blobs.Size } } - - return nil -} - -func (m *unifiedStorageMigrator) getLegacyCount(sess *xorm.Session, group, resourceType, namespace string) (int64, error) { - // Parse namespace to get org ID - orgID, err := ParseOrgIDFromNamespace(namespace) - if err != nil { - return 0, fmt.Errorf("invalid namespace %s: %w", namespace, err) - } - - // Map group/resource to legacy table - tableName, whereClause := m.getLegacyTableInfo(group, resourceType) - if tableName == "" { - return 0, fmt.Errorf("unknown resource type: %s.%s", resourceType, group) - } - - // Count items in legacy table using Table() before Count() - count, err := sess.Table(tableName).Where(whereClause, orgID).Count() - if err != nil { - return 0, fmt.Errorf("failed to count %s: %w", tableName, err) - } - - return count, nil -} - -func (m *unifiedStorageMigrator) getLegacyTableInfo(group, resource string) (table string, whereClause string) { - // Map unified storage group/resource to legacy tables - switch { - case group == "dashboard.grafana.app" && resource == "dashboards": - return "dashboard", "org_id = ? and is_folder = false" - case group == "folder.grafana.app" && resource == "folders": - return "dashboard", "org_id = ? and is_folder = true" - case group == "playlist.grafana.app" && resource == "playlists": - return "playlist", "org_id = ?" - default: - return "", "" - } -} - -func ParseOrgIDFromNamespace(namespace string) (int64, error) { - // Use authlib to properly parse all namespace formats including "default" for org 1 - info, err := types.ParseNamespace(namespace) - if err != nil { - return 0, fmt.Errorf("failed to parse namespace: %w", err) - } - return info.OrgID, nil + m.log.Info("finished migrating legacy resources", "blobStore", blobStore) + return stream.CloseAndRecv() } diff --git a/pkg/storage/unified/migrations/migrator_mock.go b/pkg/storage/unified/migrations/migrator_mock.go new file mode 100644 index 00000000000..3b460ca6e5b --- /dev/null +++ b/pkg/storage/unified/migrations/migrator_mock.go @@ -0,0 +1,98 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package migrations + +import ( + context "context" + + legacy "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" + mock "github.com/stretchr/testify/mock" + + resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb" +) + +// MockUnifiedMigrator is an autogenerated mock type for the UnifiedMigrator type +type MockUnifiedMigrator struct { + mock.Mock +} + +type MockUnifiedMigrator_Expecter struct { + mock *mock.Mock +} + +func (_m *MockUnifiedMigrator) EXPECT() *MockUnifiedMigrator_Expecter { + return &MockUnifiedMigrator_Expecter{mock: &_m.Mock} +} + +// Migrate provides a mock function with given fields: ctx, opts +func (_m *MockUnifiedMigrator) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) { + ret := _m.Called(ctx, opts) + + if len(ret) == 0 { + panic("no return value specified for Migrate") + } + + var r0 *resourcepb.BulkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)); ok { + return rf(ctx, opts) + } + if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) *resourcepb.BulkResponse); ok { + r0 = rf(ctx, opts) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*resourcepb.BulkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, legacy.MigrateOptions) error); ok { + r1 = rf(ctx, opts) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockUnifiedMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate' +type MockUnifiedMigrator_Migrate_Call struct { + *mock.Call +} + +// Migrate is a helper method to define mock.On call +// - ctx context.Context +// - opts legacy.MigrateOptions +func (_e *MockUnifiedMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockUnifiedMigrator_Migrate_Call { + return &MockUnifiedMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)} +} + +func (_c *MockUnifiedMigrator_Migrate_Call) Run(run func(ctx context.Context, opts legacy.MigrateOptions)) *MockUnifiedMigrator_Migrate_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(legacy.MigrateOptions)) + }) + return _c +} + +func (_c *MockUnifiedMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockUnifiedMigrator_Migrate_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockUnifiedMigrator_Migrate_Call) RunAndReturn(run func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)) *MockUnifiedMigrator_Migrate_Call { + _c.Call.Return(run) + return _c +} + +// NewMockUnifiedMigrator creates a new instance of MockUnifiedMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockUnifiedMigrator(t interface { + mock.TestingT + Cleanup(func()) +}) *MockUnifiedMigrator { + mock := &MockUnifiedMigrator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/storage/unified/migrations/resource_migration.go b/pkg/storage/unified/migrations/resource_migration.go new file mode 100644 index 00000000000..663bf030f67 --- /dev/null +++ b/pkg/storage/unified/migrations/resource_migration.go @@ -0,0 +1,275 @@ +package migrations + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/authlib/types" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/storage/unified/resourcepb" + "github.com/grafana/grafana/pkg/util/xorm" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// ValidationFunc is a function that validates migration results. +// It receives the database session, migration response, and logger for reporting. +// Return an error if validation fails, nil if validation passes or is skipped. +type ValidationFunc func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error + +// ResourceMigration handles migration of specific resource types from legacy to unified storage. +// It implements migrator.CodeMigration and provides a generic, extensible way to migrate any +// resource type by: +// +// 1. Iterating through all organizations +// 2. For each org, delegating to LegacyMigrator to read from legacy and write to unified storage +// 3. Validating migration results using the provided validation function (if any) +// +// To add a new resource type migration, simply create a new ResourceMigration instance in +// service.go with the appropriate schema.GroupResource specifications and optional validation function. +type ResourceMigration struct { + migrator.MigrationBase + migrator UnifiedMigrator + resources []schema.GroupResource + migrationID string + validationFunc ValidationFunc // Optional: custom validation logic for this migration + log log.Logger +} + +// NewResourceMigration creates a new migration for the specified resources. +// This is the primary way to register new resource migrations. +// +// Parameters: +// - legacyMigrator: handles reading from legacy storage and writing to unified storage +// - resources: list of GroupResource to migrate +// - migrationID: unique identifier for this migration +// - validationFunc: optional validation function to verify migration results. +// If nil, no validation will be performed. +// +// Example with legacy table count validation: +// +// NewResourceMigration( +// migrator, +// []schema.GroupResource{{Group: "playlist.grafana.app", Resource: "playlists"}}, +// "playlists", +// NewLegacyTableCountValidator(map[string]LegacyTableInfo{ +// "playlist.grafana.app/playlists": {Table: "playlist", WhereClause: "org_id = ?"}, +// }), +// ) +// +// Example without validation: +// +// NewResourceMigration(migrator, resources, "new-resource", nil) +func NewResourceMigration( + migrator UnifiedMigrator, + resources []schema.GroupResource, + migrationID string, + validationFunc ValidationFunc, +) *ResourceMigration { + return &ResourceMigration{ + migrator: migrator, + resources: resources, + migrationID: migrationID, + validationFunc: validationFunc, + log: log.New("storage.unified.resource_migration." + migrationID), + } +} + +var _ migrator.CodeMigration = (*ResourceMigration)(nil) + +// SQL implements migrator.Migration interface. Returns a description string. +func (m *ResourceMigration) SQL(_ migrator.Dialect) string { + return fmt.Sprintf("unified storage data migration: %s", m.migrationID) +} + +// Exec implements migrator.CodeMigration interface. Executes the migration across all organizations. +func (m *ResourceMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + ctx := context.Background() + + orgs, err := m.getAllOrgs(sess) + if err != nil { + m.log.Error("failed to get organizations", "error", err) + return fmt.Errorf("failed to get organizations: %w", err) + } + + if len(orgs) == 0 { + m.log.Info("No organizations found to migrate, skipping migration") + return nil + } + + m.log.Info("Starting migration for all organizations", "org_count", len(orgs), "resources", m.resources) + + for _, org := range orgs { + if err := m.migrateOrg(ctx, sess, org); err != nil { + return err + } + } + + m.log.Info("Migration completed successfully for all organizations", "org_count", len(orgs)) + return nil +} + +// migrateOrg handles migration for a single organization +func (m *ResourceMigration) migrateOrg(ctx context.Context, sess *xorm.Session, org orgInfo) error { + namespace := types.OrgNamespaceFormatter(org.ID) + m.log.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace) + + // Create a service identity context for this namespace to authenticate with unified storage + migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace) + + startTime := time.Now() + + opts := legacy.MigrateOptions{ + Namespace: namespace, + Resources: m.resources, + WithHistory: true, // Migrate with full history + Progress: func(count int, msg string) { + m.log.Info("Migration progress", "org_id", org.ID, "count", count, "message", msg) + }, + } + + // Execute the migration via legacy migrator + response, err := m.migrator.Migrate(migrationCtx, opts) + if err != nil { + m.log.Error("Migration failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime)) + return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err) + } + + // Validate the migration results + if err := m.validateMigration(sess, response); err != nil { + m.log.Error("Migration validation failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime)) + return fmt.Errorf("migration validation failed for org %d (%s): %w", org.ID, org.Name, err) + } + + m.log.Info("Migration completed for organization", + "org_id", org.ID, + "duration", time.Since(startTime), + "processed", response.Processed, + "summaries", len(response.Summary), + "rejected", len(response.Rejected)) + + return nil +} + +// validateMigration calls the custom validation function if provided +func (m *ResourceMigration) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error { + if m.validationFunc == nil { + m.log.Debug("No validation function provided, skipping validation") + return nil + } + + return m.validationFunc(sess, response, m.log) +} + +// LegacyTableInfo defines how to map a unified storage resource to its legacy table +type LegacyTableInfo struct { + Table string // Legacy table name (e.g., "dashboard", "playlist") + WhereClause string // WHERE clause template with org_id parameter (e.g., "org_id = ? and is_folder = false") +} + +// NewLegacyTableCountValidator creates a ValidationFunc that validates migration by comparing +// counts between legacy tables and unified storage. +// +// This is a helper for the common case of validating that all items from legacy tables +// were successfully migrated to unified storage. +// +// Parameters: +// - legacyTableMap: maps "group/resource" keys to LegacyTableInfo for validation. +// Only resources with mappings will be validated. +// +// Example: +// +// validator := NewLegacyTableCountValidator(map[string]LegacyTableInfo{ +// "dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"}, +// "folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"}, +// }) +func NewLegacyTableCountValidator(legacyTableMap map[string]LegacyTableInfo) ValidationFunc { + return func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error { + // Check for rejected items + if len(response.Rejected) > 0 { + log.Warn("Migration had rejected items", "count", len(response.Rejected)) + for i, rejected := range response.Rejected { + if i < 10 { // Log first 10 rejected items + log.Warn("Rejected item", + "namespace", rejected.Key.Namespace, + "group", rejected.Key.Group, + "resource", rejected.Key.Resource, + "name", rejected.Key.Name, + "reason", rejected.Error) + } + } + // Rejections are not fatal - they may be expected for invalid data + } + + // Validate counts for each resource type + for _, summary := range response.Summary { + key := fmt.Sprintf("%s/%s", summary.Group, summary.Resource) + tableInfo, ok := legacyTableMap[key] + if !ok { + log.Debug("No legacy table mapping for resource, skipping count validation", + "resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group), + "namespace", summary.Namespace) + continue + } + + // Get legacy count + orgID, err := ParseOrgIDFromNamespace(summary.Namespace) + if err != nil { + return fmt.Errorf("invalid namespace %s: %w", summary.Namespace, err) + } + + legacyCount, err := sess.Table(tableInfo.Table).Where(tableInfo.WhereClause, orgID).Count() + if err != nil { + return fmt.Errorf("failed to count %s: %w", tableInfo.Table, err) + } + + // Account for rejected items in validation + expectedCount := summary.Count + int64(len(response.Rejected)) + + log.Info("Count validation", + "resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group), + "namespace", summary.Namespace, + "legacy_count", legacyCount, + "unified_count", summary.Count, + "rejected", len(response.Rejected), + "history", summary.History) + + // Validate that we migrated all items (allowing for rejected items) + if legacyCount > expectedCount { + return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d", + summary.Resource, summary.Group, summary.Namespace, + legacyCount, summary.Count, len(response.Rejected)) + } + } + + return nil + } +} + +func ParseOrgIDFromNamespace(namespace string) (int64, error) { + // Use authlib to properly parse all namespace formats including "default" for org 1 + info, err := types.ParseNamespace(namespace) + if err != nil { + return 0, fmt.Errorf("failed to parse namespace: %w", err) + } + return info.OrgID, nil +} + +// orgInfo represents basic organization information +type orgInfo struct { + ID int64 `xorm:"id"` + Name string `xorm:"name"` +} + +// getAllOrgs retrieves all organizations from the database +func (m *ResourceMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) { + var orgs []orgInfo + err := sess.Table("org").Cols("id", "name").Find(&orgs) + if err != nil { + return nil, err + } + return orgs, nil +} diff --git a/pkg/storage/unified/migrations/service.go b/pkg/storage/unified/migrations/service.go new file mode 100644 index 00000000000..5bcb2fdf8c2 --- /dev/null +++ b/pkg/storage/unified/migrations/service.go @@ -0,0 +1,118 @@ +package migrations + +import ( + "context" + "fmt" + "os" + + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/kvstore" + "github.com/grafana/grafana/pkg/infra/log" + sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/storage/unified/migrations/contract" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations") +var logger = log.New("storage.unified.migrations") + +type UnifiedStorageMigrationServiceImpl struct { + migrator UnifiedMigrator + cfg *setting.Cfg + sqlStore db.DB + kv kvstore.KVStore +} + +var _ contract.UnifiedStorageMigrationService = (*UnifiedStorageMigrationServiceImpl)(nil) + +// ProvideUnifiedStorageMigrationService is a Wire provider that creates the migration service. +// The service implements registry.BackgroundService and runs migrations during server startup. +func ProvideUnifiedStorageMigrationService( + migrator UnifiedMigrator, + cfg *setting.Cfg, + sqlStore db.DB, + kv kvstore.KVStore, +) contract.UnifiedStorageMigrationService { + return &UnifiedStorageMigrationServiceImpl{ + migrator: migrator, + cfg: cfg, + sqlStore: sqlStore, + kv: kv, + } +} + +// Run executes unified storage migrations as a background service. +// This blocks until migrations complete. If migrations fail, an error is returned +// which will prevent Grafana from starting. +func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error { + // TODO: temporary skip migrations in test environments to prevent integration test timeouts. + if os.Getenv("GRAFANA_TEST_DB") != "" { + return nil + } + + // skip migrations if disabled in config + if p.cfg.DisableDataMigrations { + logger.Info("Data migrations are disabled, skipping") + return nil + } + + // TODO: Re-enable once migrations are ready + // TODO: add guarantee that this only runs once + // return RegisterMigrations(p.migrator, p.cfg, p.sqlStore) + return nil +} + +// RegisterMigrations initializes and registers all unified storage migrations. +// This function is the entry point for all data migrations from legacy storage +// to unified storage. It returns an error if migrations fail, preventing Grafana +// from starting with inconsistent data. +func RegisterMigrations( + migrator UnifiedMigrator, + cfg *setting.Cfg, + sqlStore db.DB, +) error { + ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations") + defer span.End() + mg := sqlstoremigrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unifiedstorage") + mg.AddCreateMigration() + + if err := prometheus.Register(mg); err != nil { + logger.Warn("Failed to register migrator metrics", "error", err) + } + + // Register resource migrations + // To add a new resource type, simply add another migration here with the appropriate resources + registerResourceMigrations(mg, migrator) + + // Run all registered migrations (blocking) + sec := cfg.Raw.Section("database") + if err := mg.RunMigrations(ctx, + sec.Key("migration_locking").MustBool(true), + sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil { + return fmt.Errorf("unified storage data migration failed: %w", err) + } + + logger.Info("Unified storage migrations completed successfully") + return nil +} + +// registerResourceMigrations registers all unified storage resource migrations. +// Add new resource types here by creating additional ResourceMigration instances. +func registerResourceMigrations(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator) { + dashboardsAndFolders := NewResourceMigration( + migrator, + []schema.GroupResource{ + {Group: "folder.grafana.app", Resource: "folders"}, + {Group: "dashboard.grafana.app", Resource: "dashboards"}, + }, + "folders-dashboards", + NewLegacyTableCountValidator(map[string]LegacyTableInfo{ + "folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"}, + "dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"}, + }), + ) + mg.AddMigration("folders and dashboards migration", dashboardsAndFolders) +}