feat: inject unified data migrations in dual writer (#114138)

* feat: draft changes for on-prem unified migration

* feat: further draft changes for on-prem unified migration

* fix: remove some tbis

* refactor: rename

* fix: another approach

* fix: background service related issues

* fix: address comments

* fix: make gen-go

* fix: background service related issues

* feat: refactor dual writer and legacy migrator

* fix: minor issues

* feat: working version in oss

* fix: wire

* fix: revert test data override

* fix: enterprise related issues

* chore: add todo

* fix: revert dual writer method

* fix: lint

* chore: logger format

* fix: reduce log level

* fix: log change

* fix: disable

* fix: address comments

* fix: return error on dual writer service

* fix: merge conflict

---------

Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
Mustafa Sencer Özcan
2025-11-20 16:40:20 +01:00
committed by GitHub
co-authored by Rafael Paulovic
parent cb06bba243
commit 30c04ab3fc
40 changed files with 1779 additions and 1348 deletions
@@ -1,70 +0,0 @@
package datamigrations
import (
"context"
"path/filepath"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
)
var (
_ provisioning.ProvisioningService = (*stubProvisioning)(nil)
)
func newStubProvisioning(path string) (provisioning.ProvisioningService, error) {
cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
if err != nil {
return nil, err
}
stub := &stubProvisioning{
path: make(map[string]string),
}
for _, cfg := range cfgs {
stub.path[cfg.Name] = cfg.Options["path"].(string)
}
return &stubProvisioning{}, nil
}
type stubProvisioning struct {
path map[string]string // name > options.path
}
// GetAllowUIUpdatesFromConfig implements provisioning.ProvisioningService.
func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
return false
}
func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
return s.path[name]
}
// ProvisionAlerting implements provisioning.ProvisioningService.
func (s *stubProvisioning) ProvisionAlerting(ctx context.Context) error {
panic("unimplemented")
}
// ProvisionDashboards implements provisioning.ProvisioningService.
func (s *stubProvisioning) ProvisionDashboards(ctx context.Context) error {
panic("unimplemented")
}
// ProvisionDatasources implements provisioning.ProvisioningService.
func (s *stubProvisioning) ProvisionDatasources(ctx context.Context) error {
panic("unimplemented")
}
// ProvisionPlugins implements provisioning.ProvisioningService.
func (s *stubProvisioning) ProvisionPlugins(ctx context.Context) error {
panic("unimplemented")
}
// Run implements provisioning.ProvisioningService.
func (s *stubProvisioning) Run(ctx context.Context) error {
panic("unimplemented")
}
// RunInitProvisioners implements provisioning.ProvisioningService.
func (s *stubProvisioning) RunInitProvisioners(ctx context.Context) error {
panic("unimplemented")
}
@@ -24,10 +24,11 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/accesscontrol/acimpl"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/search/sort"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified"
"github.com/grafana/grafana/pkg/storage/unified/migrations"
"github.com/grafana/grafana/pkg/storage/unified/parquet"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
@@ -52,7 +53,6 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
{Group: folders.GROUP, Resource: folders.RESOURCE},
{Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE},
},
LargeObjects: nil, // TODO... from config
Progress: func(count int, msg string) {
const minInterval = time.Second
shouldPrint := count < 1 || time.Since(last) > minInterval
@@ -69,31 +69,26 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
}
featureToggles := featuremgmt.ProvideToggles(featureManager)
provisioning, err := newStubProvisioning(cfg.ProvisioningPath)
provisioning, err := provisioning.ProvideStubProvisioningService(cfg)
if err != nil {
return err
}
migrator := legacy.NewDashboardAccess(
grpcClient, err := newUnifiedClient(cfg, sqlStore, featureToggles)
if err != nil {
return err
}
dashboardAccess := legacy.ProvideMigratorDashboardAccessor(
legacysql.NewDatabaseProvider(sqlStore),
authlib.OrgNamespaceFormatter,
nil, // no dashboards.Store
provisioning,
nil, // no librarypanels.Service
sort.ProvideService(),
nil, // we don't delete during migration, and this is only need to delete permission.
acimpl.ProvideAccessControl(featuremgmt.WithFeatures()),
featureToggles,
)
client, err := newUnifiedClient(cfg, sqlStore, featureToggles)
if err != nil {
return err
}
if c.Bool("non-interactive") {
opts.Store = client
opts.BlobStore = client
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
opts.WithHistory = true // always include history in non-interactive mode
rsp, err := migrator.Migrate(ctx, opts)
if exitErr := handleMigrationError(err, rsp); exitErr != nil {
@@ -113,6 +108,8 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
return err
}
if yes {
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, nil) // no need for grpc client for counting
opts.OnlyCount = true
rsp, err := migrator.Migrate(ctx, opts)
if err != nil {
@@ -141,12 +138,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
if err != nil {
return err
}
start = time.Now()
last = time.Now()
opts.Store, err = newParquetClient(file)
parquetClient, err := newParquetClient(file)
if err != nil {
return err
}
migrator := migrations.ProvideUnifiedMigratorParquet(dashboardAccess, parquetClient)
start = time.Now()
last = time.Now()
rsp, err := migrator.Migrate(ctx, opts)
if err != nil {
return err
@@ -172,7 +170,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
req.Kinds = append(req.Kinds, fmt.Sprintf("%s/%s", r.Group, r.Resource))
}
stats, err := client.GetStats(ctx, req)
stats, err := grpcClient.GetStats(ctx, req)
if err != nil {
return err
}
@@ -188,10 +186,9 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
return err
}
if yes {
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
start = time.Now()
last = time.Now()
opts.Store = client
opts.BlobStore = client
rsp, err := migrator.Migrate(ctx, opts)
if err != nil {
return err
@@ -1,96 +0,0 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
package legacy
import (
context "context"
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
mock "github.com/stretchr/testify/mock"
)
// MockLegacyMigrator is an autogenerated mock type for the LegacyMigrator type
type MockLegacyMigrator struct {
mock.Mock
}
type MockLegacyMigrator_Expecter struct {
mock *mock.Mock
}
func (_m *MockLegacyMigrator) EXPECT() *MockLegacyMigrator_Expecter {
return &MockLegacyMigrator_Expecter{mock: &_m.Mock}
}
// Migrate provides a mock function with given fields: ctx, opts
func (_m *MockLegacyMigrator) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
ret := _m.Called(ctx, opts)
if len(ret) == 0 {
panic("no return value specified for Migrate")
}
var r0 *resourcepb.BulkResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
return rf(ctx, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
r0 = rf(ctx, opts)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*resourcepb.BulkResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
r1 = rf(ctx, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockLegacyMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
type MockLegacyMigrator_Migrate_Call struct {
*mock.Call
}
// Migrate is a helper method to define mock.On call
// - ctx context.Context
// - opts MigrateOptions
func (_e *MockLegacyMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockLegacyMigrator_Migrate_Call {
return &MockLegacyMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
}
func (_c *MockLegacyMigrator_Migrate_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockLegacyMigrator_Migrate_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(MigrateOptions))
})
return _c
}
func (_c *MockLegacyMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockLegacyMigrator_Migrate_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockLegacyMigrator_Migrate_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockLegacyMigrator_Migrate_Call {
_c.Call.Return(run)
return _c
}
// NewMockLegacyMigrator creates a new instance of MockLegacyMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockLegacyMigrator(t interface {
mock.TestingT
Cleanup(func())
}) *MockLegacyMigrator {
mock := &MockLegacyMigrator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -1,460 +0,0 @@
package legacy
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"google.golang.org/grpc/metadata"
"k8s.io/apimachinery/pkg/runtime/schema"
authlib "github.com/grafana/authlib/types"
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/librarypanels"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/search/sort"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified/apistore"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
type MigrateOptions struct {
Namespace string
Store resourcepb.BulkStoreClient
LargeObjects apistore.LargeObjectSupport
BlobStore resourcepb.BlobStoreClient
Resources []schema.GroupResource
WithHistory bool // only applies to dashboards
OnlyCount bool // just count the values
Progress func(count int, msg string)
}
// Read from legacy and write into unified storage
//
//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter
type LegacyMigrator interface {
Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
}
// This can migrate Folders, Dashboards and LibraryPanels
func ProvideLegacyMigrator(
sql db.DB, // direct access to tables
provisioning provisioning.ProvisioningService, // only needed for dashboard settings
libraryPanelSvc librarypanels.Service,
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
accessControl accesscontrol.AccessControl,
features featuremgmt.FeatureToggles,
) LegacyMigrator {
dbp := legacysql.NewDatabaseProvider(sql)
return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), dashboardPermissionSvc, accessControl, features)
}
type BlobStoreInfo struct {
Count int64
Size int64
}
// migrate function -- works for a single kind
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
info, err := authlib.ParseNamespace(opts.Namespace)
if err != nil {
return nil, err
}
if opts.Progress == nil {
opts.Progress = func(count int, msg string) {} // noop
}
// Migrate everything
if len(opts.Resources) < 1 {
return nil, fmt.Errorf("missing resource selector")
}
migratorFuncs := []migratorFunc{}
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
migratorFuncs = append(migratorFuncs, a.migrateFolders)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
Resource: folders.RESOURCE,
})
case "dashboard.grafana.app/librarypanels":
migratorFuncs = append(migratorFuncs, a.migratePanels)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
})
case "dashboard.grafana.app/dashboards":
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
})
default:
return nil, fmt.Errorf("unsupported resource: %s", res)
}
}
if opts.OnlyCount {
return a.countValues(ctx, opts)
}
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
if md, ok := metadata.FromOutgoingContext(ctx); ok {
a.log.Debug("bulk grpc request metadata",
"metadata", md,
"collection", settings.Collection,
)
} else {
a.log.Debug("bulk grpc request, no metadata found",
"collection", settings.Collection,
)
}
stream, err := opts.Store.BulkProcess(ctx)
if err != nil {
return nil, err
}
// Now run each migration
blobStore := BlobStoreInfo{}
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
for _, m := range migratorFuncs {
blobs, err := m(ctx, info.OrgID, opts, stream)
if err != nil {
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
return nil, err
}
if blobs != nil {
blobStore.Count += blobs.Count
blobStore.Size += blobs.Size
}
}
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
return stream.CloseAndRecv()
}
func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
sql, err := a.sql(ctx)
if err != nil {
return nil, err
}
ns, err := authlib.ParseNamespace(opts.Namespace)
if err != nil {
return nil, err
}
orgId := ns.OrgID
rsp := &resourcepb.BulkResponse{}
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = folders.GROUP
summary.Group = folders.RESOURCE
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
rsp.Summary = append(rsp.Summary, summary)
case "dashboard.grafana.app/librarypanels":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = dashboard.GROUP
summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
" WHERE org_id=?", orgId).Get(&summary.Count)
rsp.Summary = append(rsp.Summary, summary)
case "dashboard.grafana.app/dashboards":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = dashboard.GROUP
summary.Resource = dashboard.DASHBOARD_RESOURCE
rsp.Summary = append(rsp.Summary, summary)
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
if err != nil {
return err
}
// Also count history
_, err = sess.SQL(`SELECT COUNT(*)
FROM `+sql.Table("dashboard_version")+` as dv
JOIN `+sql.Table("dashboard")+` as dd
ON dd.id = dv.dashboard_id
WHERE org_id=?`, orgId).Get(&summary.History)
}
if err != nil {
return err
}
}
return nil
})
return rsp, nil
}
func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
query := &DashboardQuery{
OrgID: orgId,
Limit: 100000000,
GetHistory: opts.WithHistory, // include history
AllowFallback: true, // allow fallback to dashboard table during migration
Order: "ASC", // oldest first
}
blobs := &BlobStoreInfo{}
sql, err := a.sql(ctx)
if err != nil {
return blobs, err
}
opts.Progress(-1, "migrating dashboards...")
rows, err := a.getRows(ctx, sql, query)
if rows != nil {
defer func() {
_ = rows.Close()
}()
}
if err != nil {
return blobs, err
}
large := opts.LargeObjects
// Now send each dashboard
for i := 1; rows.Next(); i++ {
dash := rows.row.Dash
if dash.APIVersion == "" {
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP)
}
dash.SetNamespace(opts.Namespace)
dash.SetResourceVersion("") // it will be filled in by the backend
body, err := json.Marshal(dash)
if err != nil {
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
return blobs, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
Name: rows.Name(),
},
Value: body,
Folder: rows.row.FolderUID,
Action: resourcepb.BulkRequest_ADDED,
}
if dash.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
} else if dash.Generation < 0 {
req.Action = resourcepb.BulkRequest_DELETED
}
// With large object support
if large != nil && len(body) > large.Threshold() {
obj, err := utils.MetaAccessor(dash)
if err != nil {
return blobs, err
}
opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body)))
err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value)
if err != nil {
return blobs, err
}
// The smaller version (most of spec removed)
req.Value, err = json.Marshal(dash)
if err != nil {
return blobs, err
}
blobs.Count++
blobs.Size += int64(len(body))
}
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
err = nil
}
return blobs, err
}
}
if len(rows.rejected) > 0 {
for _, row := range rows.rejected {
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
a.log.Warn("rejected dashboard",
"namespace", opts.Namespace,
"dashboard", row.Dash.Name,
"uid", row.Dash.UID,
"id", id,
"version", row.Dash.Generation,
)
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
}
}
if rows.Error() != nil {
return blobs, rows.Error()
}
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
return blobs, err
}
func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
query := &DashboardQuery{
OrgID: orgId,
Limit: 100000000,
GetFolders: true,
Order: "ASC",
}
sql, err := a.sql(ctx)
if err != nil {
return nil, err
}
opts.Progress(-1, "migrating folders...")
rows, err := a.getRows(ctx, sql, query)
if rows != nil {
defer func() {
_ = rows.Close()
}()
}
if err != nil {
return nil, err
}
// Now send each dashboard
for i := 1; rows.Next(); i++ {
dash := rows.row.Dash
dash.APIVersion = "folder.grafana.app/v1beta1"
dash.Kind = "Folder"
dash.SetNamespace(opts.Namespace)
dash.SetResourceVersion("") // it will be filled in by the backend
spec := map[string]any{
"title": dash.Spec.Object["title"],
}
description := dash.Spec.Object["description"]
if description != nil {
spec["description"] = description
}
dash.Spec.Object = spec
body, err := json.Marshal(dash)
if err != nil {
return nil, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: "folder.grafana.app",
Resource: "folders",
Name: rows.Name(),
},
Value: body,
Folder: rows.row.FolderUID,
Action: resourcepb.BulkRequest_ADDED,
}
if dash.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
} else if dash.Generation < 0 {
req.Action = resourcepb.BulkRequest_DELETED
}
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return nil, err
}
}
if rows.Error() != nil {
return nil, rows.Error()
}
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
return nil, err
}
func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
opts.Progress(-1, "migrating library panels...")
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
OrgID: orgId,
Limit: 1000000,
})
if err != nil {
return nil, err
}
for i, panel := range panels.Items {
meta, err := utils.MetaAccessor(&panel)
if err != nil {
return nil, err
}
body, err := json.Marshal(panel)
if err != nil {
return nil, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboard.GROUP,
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
Name: panel.Name,
},
Value: body,
Folder: meta.GetFolder(),
Action: resourcepb.BulkRequest_ADDED,
}
if panel.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
}
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return nil, err
}
}
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
return nil, nil
}
@@ -1,154 +0,0 @@
package legacy
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
)
func TestDashboardMigrationQuery(t *testing.T) {
// Test that migration queries use AllowFallback flag correctly
nodb := &legacysql.LegacyDatabaseHelper{
Table: func(n string) string {
return "grafana." + n
},
}
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
// Create a migration query as would be used in actual migration
migrationQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true, // Migration includes history
AllowFallback: true, // This is the key flag for migration
Order: "ASC", // Migration uses ascending order
}
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
// Verify the flag is set correctly
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
})
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
// Regular history query without migration
historyQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
Order: "DESC",
}
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
require.True(t, historyQuery.GetHistory, "History query should get history")
})
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
// Test that the SQL template produces COALESCE logic for migration queries
migrationQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
AllowFallback: true,
Order: "ASC",
}
req := newQueryReq(nodb, migrationQuery)
req.SQLTemplate = mocks.NewTestingSQLTemplate()
// Execute the template to get the generated SQL
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
require.NoError(t, err)
sql := rawQuery
// Verify that COALESCE functions are present in the generated SQL
// These should be used when GetHistory=true AND AllowFallback=true
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
"Migration SQL should contain COALESCE for updated timestamp")
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
"Migration SQL should contain COALESCE for version")
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
"Migration SQL should contain COALESCE for data")
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
"Migration SQL should contain COALESCE for api_version")
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
"Migration SQL should contain COALESCE for message with empty string fallback")
// Verify ORDER BY uses COALESCE as well
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
"Migration SQL should ORDER BY COALESCED created timestamp")
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
"Migration SQL should ORDER BY COALESCED version")
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
"Migration SQL should not exclude dashboards without version entries")
})
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
// Test that regular history queries still use strict dashboard_version fields
historyQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
Order: "DESC",
}
req := newQueryReq(nodb, historyQuery)
req.SQLTemplate = mocks.NewTestingSQLTemplate()
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
require.NoError(t, err)
sql := rawQuery
// Verify that direct dashboard_version fields are used (no COALESCE)
require.Contains(t, sql, "dashboard_version.created as updated",
"Regular history SQL should use direct dashboard_version.created")
require.Contains(t, sql, "dashboard_version.version",
"Regular history SQL should use direct dashboard_version.version")
require.Contains(t, sql, "dashboard_version.data",
"Regular history SQL should use direct dashboard_version.data")
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
// to allow for cases where dashboard_version entries might be missing
// Should not contain COALESCE functions
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
"Regular history SQL should not contain COALESCE for updated")
})
}
func TestMigrateDashboardsConfiguration(t *testing.T) {
// Test the actual migration function configuration
t.Run("Migration options should configure query correctly", func(t *testing.T) {
// Test the migration configuration as used in real migration
opts := MigrateOptions{
WithHistory: true, // Migration includes history
}
// This simulates what happens in migrateDashboards function
expectedQuery := &DashboardQuery{
OrgID: 1,
Limit: 100000000,
GetHistory: opts.WithHistory, // Should be true
AllowFallback: true, // Should be true for migration
Order: "ASC", // Should be ASC for migration
}
// Verify the configuration matches what migration sets up
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
// Verify UseHistoryTable logic
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
})
}
@@ -0,0 +1,279 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
package legacy
import (
context "context"
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
mock "github.com/stretchr/testify/mock"
)
// MockMigrationDashboardAccessor is an autogenerated mock type for the MigrationDashboardAccessor type
type MockMigrationDashboardAccessor struct {
mock.Mock
}
type MockMigrationDashboardAccessor_Expecter struct {
mock *mock.Mock
}
func (_m *MockMigrationDashboardAccessor) EXPECT() *MockMigrationDashboardAccessor_Expecter {
return &MockMigrationDashboardAccessor_Expecter{mock: &_m.Mock}
}
// CountResources provides a mock function with given fields: ctx, opts
func (_m *MockMigrationDashboardAccessor) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
ret := _m.Called(ctx, opts)
if len(ret) == 0 {
panic("no return value specified for CountResources")
}
var r0 *resourcepb.BulkResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
return rf(ctx, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
r0 = rf(ctx, opts)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*resourcepb.BulkResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
r1 = rf(ctx, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMigrationDashboardAccessor_CountResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountResources'
type MockMigrationDashboardAccessor_CountResources_Call struct {
*mock.Call
}
// CountResources is a helper method to define mock.On call
// - ctx context.Context
// - opts MigrateOptions
func (_e *MockMigrationDashboardAccessor_Expecter) CountResources(ctx interface{}, opts interface{}) *MockMigrationDashboardAccessor_CountResources_Call {
return &MockMigrationDashboardAccessor_CountResources_Call{Call: _e.mock.On("CountResources", ctx, opts)}
}
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockMigrationDashboardAccessor_CountResources_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(MigrateOptions))
})
return _c
}
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockMigrationDashboardAccessor_CountResources_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMigrationDashboardAccessor_CountResources_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockMigrationDashboardAccessor_CountResources_Call {
_c.Call.Return(run)
return _c
}
// MigrateDashboards provides a mock function with given fields: ctx, orgId, opts, stream
func (_m *MockMigrationDashboardAccessor) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
ret := _m.Called(ctx, orgId, opts, stream)
if len(ret) == 0 {
panic("no return value specified for MigrateDashboards")
}
var r0 *BlobStoreInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
return rf(ctx, orgId, opts, stream)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
r0 = rf(ctx, orgId, opts, stream)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*BlobStoreInfo)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
r1 = rf(ctx, orgId, opts, stream)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMigrationDashboardAccessor_MigrateDashboards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateDashboards'
type MockMigrationDashboardAccessor_MigrateDashboards_Call struct {
*mock.Call
}
// MigrateDashboards is a helper method to define mock.On call
// - ctx context.Context
// - orgId int64
// - opts MigrateOptions
// - stream resourcepb.BulkStore_BulkProcessClient
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateDashboards(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
return &MockMigrationDashboardAccessor_MigrateDashboards_Call{Call: _e.mock.On("MigrateDashboards", ctx, orgId, opts, stream)}
}
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
})
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
_c.Call.Return(run)
return _c
}
// MigrateFolders provides a mock function with given fields: ctx, orgId, opts, stream
func (_m *MockMigrationDashboardAccessor) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
ret := _m.Called(ctx, orgId, opts, stream)
if len(ret) == 0 {
panic("no return value specified for MigrateFolders")
}
var r0 *BlobStoreInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
return rf(ctx, orgId, opts, stream)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
r0 = rf(ctx, orgId, opts, stream)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*BlobStoreInfo)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
r1 = rf(ctx, orgId, opts, stream)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMigrationDashboardAccessor_MigrateFolders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateFolders'
type MockMigrationDashboardAccessor_MigrateFolders_Call struct {
*mock.Call
}
// MigrateFolders is a helper method to define mock.On call
// - ctx context.Context
// - orgId int64
// - opts MigrateOptions
// - stream resourcepb.BulkStore_BulkProcessClient
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateFolders(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateFolders_Call {
return &MockMigrationDashboardAccessor_MigrateFolders_Call{Call: _e.mock.On("MigrateFolders", ctx, orgId, opts, stream)}
}
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
})
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateFolders_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
_c.Call.Return(run)
return _c
}
// MigrateLibraryPanels provides a mock function with given fields: ctx, orgId, opts, stream
func (_m *MockMigrationDashboardAccessor) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
ret := _m.Called(ctx, orgId, opts, stream)
if len(ret) == 0 {
panic("no return value specified for MigrateLibraryPanels")
}
var r0 *BlobStoreInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
return rf(ctx, orgId, opts, stream)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
r0 = rf(ctx, orgId, opts, stream)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*BlobStoreInfo)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
r1 = rf(ctx, orgId, opts, stream)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMigrationDashboardAccessor_MigrateLibraryPanels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateLibraryPanels'
type MockMigrationDashboardAccessor_MigrateLibraryPanels_Call struct {
*mock.Call
}
// MigrateLibraryPanels is a helper method to define mock.On call
// - ctx context.Context
// - orgId int64
// - opts MigrateOptions
// - stream resourcepb.BulkStore_BulkProcessClient
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateLibraryPanels(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
return &MockMigrationDashboardAccessor_MigrateLibraryPanels_Call{Call: _e.mock.On("MigrateLibraryPanels", ctx, orgId, opts, stream)}
}
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
})
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
_c.Call.Return(run)
return _c
}
// NewMockMigrationDashboardAccessor creates a new instance of MockMigrationDashboardAccessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockMigrationDashboardAccessor(t interface {
mock.TestingT
Cleanup(func())
}) *MockMigrationDashboardAccessor {
mock := &MockMigrationDashboardAccessor{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -4,7 +4,9 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
"sync"
@@ -13,6 +15,7 @@ import (
"go.opentelemetry.io/otel"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/ptr"
claims "github.com/grafana/authlib/types"
@@ -20,6 +23,7 @@ import (
dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
@@ -35,16 +39,30 @@ import (
"github.com/grafana/grafana/pkg/services/librarypanels"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/search/sort"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
var (
_ DashboardAccess = (*dashboardSqlAccess)(nil)
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
)
type MigrateOptions struct {
Namespace string
Resources []schema.GroupResource
WithHistory bool // only applies to dashboards
OnlyCount bool // just count the values
Progress func(count int, msg string)
}
type BlobStoreInfo struct {
Count int64
Size int64
}
type dashboardRow struct {
// The numeric version for this dashboard
RV int64
@@ -63,8 +81,9 @@ type dashboardRow struct {
type dashboardSqlAccess struct {
sql legacysql.LegacyDatabaseProvider
namespacer request.NamespaceMapper
provisioning provisioning.ProvisioningService
provisioning provisioning.StubProvisioningService
// TODO: consider enabling this by default for on-prem migrations
invalidDashboardParseFallbackEnabled bool
// Use for writing (not reading)
@@ -73,7 +92,7 @@ type dashboardSqlAccess struct {
dashboardPermissionSvc accesscontrol.DashboardPermissionsService
accessControl accesscontrol.AccessControl
libraryPanelSvc librarypanels.Service
libraryPanelSvc librarypanels.Service // only used for save dashboard
// Typically one... the server wrapper
subscribers []chan *resource.WrittenEvent
@@ -81,7 +100,27 @@ type dashboardSqlAccess struct {
log log.Logger
}
func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
// ProvideMigratorDashboardAccessor creates a DashboardAccess specifically for migration purposes.
// This provider is used by Wire DI and only includes the minimal dependencies needed for migrations.
func ProvideMigratorDashboardAccessor(
sql legacysql.LegacyDatabaseProvider,
provisioning provisioning.StubProvisioningService,
accessControl accesscontrol.AccessControl,
features featuremgmt.FeatureToggles,
) MigrationDashboardAccessor {
return &dashboardSqlAccess{
sql: sql,
namespacer: claims.OrgNamespaceFormatter,
dashStore: nil, // not needed for migration
provisioning: provisioning,
dashboardPermissionSvc: nil, // not needed for migration
libraryPanelSvc: nil, // not needed for migration
accessControl: accessControl,
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
}
}
func NewDashboardSQLAccess(sql legacysql.LegacyDatabaseProvider,
namespacer request.NamespaceMapper,
dashStore dashboards.Store,
provisioning provisioning.ProvisioningService,
@@ -90,7 +129,7 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
accessControl accesscontrol.AccessControl,
features featuremgmt.FeatureToggles,
) DashboardAccess {
) *dashboardSqlAccess {
dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter)
return &dashboardSqlAccess{
sql: sql,
@@ -101,7 +140,6 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
dashboardPermissionSvc: dashboardPermissionSvc,
libraryPanelSvc: libraryPanelSvc,
accessControl: accessControl,
log: log.New("dashboard.legacysql"),
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
}
}
@@ -149,6 +187,290 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
}, err
}
// CountResources counts resources without migrating them
func (a *dashboardSqlAccess) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
sql, err := a.sql(ctx)
if err != nil {
return nil, err
}
ns, err := claims.ParseNamespace(opts.Namespace)
if err != nil {
return nil, err
}
orgId := ns.OrgID
rsp := &resourcepb.BulkResponse{}
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = folders.GROUP
summary.Group = folders.RESOURCE
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
rsp.Summary = append(rsp.Summary, summary)
case "dashboard.grafana.app/librarypanels":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = dashboardV1.GROUP
summary.Resource = dashboardV1.LIBRARY_PANEL_RESOURCE
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
" WHERE org_id=?", orgId).Get(&summary.Count)
rsp.Summary = append(rsp.Summary, summary)
case "dashboard.grafana.app/dashboards":
summary := &resourcepb.BulkResponse_Summary{}
summary.Group = dashboardV1.GROUP
summary.Resource = dashboardV1.DASHBOARD_RESOURCE
rsp.Summary = append(rsp.Summary, summary)
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
if err != nil {
return err
}
// Also count history
_, err = sess.SQL(`SELECT COUNT(*)
FROM `+sql.Table("dashboard_version")+` as dv
JOIN `+sql.Table("dashboard")+` as dd
ON dd.id = dv.dashboard_id
WHERE org_id=?`, orgId).Get(&summary.History)
}
if err != nil {
return err
}
}
return nil
})
return rsp, nil
}
// MigrateDashboards handles the dashboard migration logic
func (a *dashboardSqlAccess) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
query := &DashboardQuery{
OrgID: orgId,
Limit: 100000000,
GetHistory: opts.WithHistory, // include history
AllowFallback: true, // allow fallback to dashboard table during migration
Order: "ASC", // oldest first
}
blobs := &BlobStoreInfo{}
sql, err := a.sql(ctx)
if err != nil {
return blobs, err
}
opts.Progress(-1, "migrating dashboards...")
rows, err := a.getRows(ctx, sql, query)
if rows != nil {
defer func() {
_ = rows.Close()
}()
}
if err != nil {
return blobs, err
}
// Now send each dashboard
for i := 1; rows.Next(); i++ {
dash := rows.row.Dash
if dash.APIVersion == "" {
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboardV1.GROUP)
}
dash.SetNamespace(opts.Namespace)
dash.SetResourceVersion("") // it will be filled in by the backend
body, err := json.Marshal(dash)
if err != nil {
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
return blobs, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboardV1.GROUP,
Resource: dashboardV1.DASHBOARD_RESOURCE,
Name: rows.Name(),
},
Value: body,
Folder: rows.row.FolderUID,
Action: resourcepb.BulkRequest_ADDED,
}
if dash.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
} else if dash.Generation < 0 {
req.Action = resourcepb.BulkRequest_DELETED
}
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
err = nil
}
return blobs, err
}
}
if len(rows.rejected) > 0 {
for _, row := range rows.rejected {
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
a.log.Warn("rejected dashboard",
"namespace", opts.Namespace,
"dashboard", row.Dash.Name,
"uid", row.Dash.UID,
"id", id,
"version", row.Dash.Generation,
)
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
}
}
if rows.Error() != nil {
return blobs, rows.Error()
}
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
return blobs, err
}
// MigrateFolders handles the folder migration logic
func (a *dashboardSqlAccess) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
query := &DashboardQuery{
OrgID: orgId,
Limit: 100000000,
GetFolders: true,
Order: "ASC",
}
sql, err := a.sql(ctx)
if err != nil {
return nil, err
}
opts.Progress(-1, "migrating folders...")
rows, err := a.getRows(ctx, sql, query)
if rows != nil {
defer func() {
_ = rows.Close()
}()
}
if err != nil {
return nil, err
}
// Now send each dashboard
for i := 1; rows.Next(); i++ {
dash := rows.row.Dash
dash.APIVersion = "folder.grafana.app/v1beta1"
dash.Kind = "Folder"
dash.SetNamespace(opts.Namespace)
dash.SetResourceVersion("") // it will be filled in by the backend
spec := map[string]any{
"title": dash.Spec.Object["title"],
}
description := dash.Spec.Object["description"]
if description != nil {
spec["description"] = description
}
dash.Spec.Object = spec
body, err := json.Marshal(dash)
if err != nil {
return nil, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: "folder.grafana.app",
Resource: "folders",
Name: rows.Name(),
},
Value: body,
Folder: rows.row.FolderUID,
Action: resourcepb.BulkRequest_ADDED,
}
if dash.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
} else if dash.Generation < 0 {
req.Action = resourcepb.BulkRequest_DELETED
}
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return nil, err
}
}
if rows.Error() != nil {
return nil, rows.Error()
}
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
return nil, err
}
// MigrateLibraryPanels handles the library panel migration logic
func (a *dashboardSqlAccess) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
opts.Progress(-1, "migrating library panels...")
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
OrgID: orgId,
Limit: 1000000,
})
if err != nil {
return nil, err
}
for i, panel := range panels.Items {
meta, err := utils.MetaAccessor(&panel)
if err != nil {
return nil, err
}
body, err := json.Marshal(panel)
if err != nil {
return nil, err
}
req := &resourcepb.BulkRequest{
Key: &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: dashboardV1.GROUP,
Resource: dashboardV1.LIBRARY_PANEL_RESOURCE,
Name: panel.Name,
},
Value: body,
Folder: meta.GetFolder(),
Action: resourcepb.BulkRequest_ADDED,
}
if panel.Generation > 1 {
req.Action = resourcepb.BulkRequest_MODIFIED
}
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
err = stream.Send(req)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return nil, err
}
}
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
return nil, nil
}
var _ resource.ListIterator = (*rowsWrapper)(nil)
type rowsWrapper struct {
@@ -21,6 +21,9 @@ import (
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
)
func TestScanRow(t *testing.T) {
@@ -529,3 +532,146 @@ func TestParseLibraryPanelRow(t *testing.T) {
require.Nil(t, updatedTimestamp)
})
}
func TestDashboardMigrationQuery(t *testing.T) {
// Test that migration queries use AllowFallback flag correctly
nodb := &legacysql.LegacyDatabaseHelper{
Table: func(n string) string {
return "grafana." + n
},
}
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
// Create a migration query as would be used in actual migration
migrationQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true, // Migration includes history
AllowFallback: true, // This is the key flag for migration
Order: "ASC", // Migration uses ascending order
}
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
// Verify the flag is set correctly
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
})
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
// Regular history query without migration
historyQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
Order: "DESC",
}
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
require.True(t, historyQuery.GetHistory, "History query should get history")
})
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
// Test that the SQL template produces COALESCE logic for migration queries
migrationQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
AllowFallback: true,
Order: "ASC",
}
req := newQueryReq(nodb, migrationQuery)
req.SQLTemplate = mocks.NewTestingSQLTemplate()
// Execute the template to get the generated SQL
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
require.NoError(t, err)
sql := rawQuery
// Verify that COALESCE functions are present in the generated SQL
// These should be used when GetHistory=true AND AllowFallback=true
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
"Migration SQL should contain COALESCE for updated timestamp")
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
"Migration SQL should contain COALESCE for version")
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
"Migration SQL should contain COALESCE for data")
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
"Migration SQL should contain COALESCE for api_version")
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
"Migration SQL should contain COALESCE for message with empty string fallback")
// Verify ORDER BY uses COALESCE as well
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
"Migration SQL should ORDER BY COALESCED created timestamp")
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
"Migration SQL should ORDER BY COALESCED version")
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
"Migration SQL should not exclude dashboards without version entries")
})
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
// Test that regular history queries still use strict dashboard_version fields
historyQuery := &DashboardQuery{
OrgID: 1,
GetHistory: true,
Order: "DESC",
}
req := newQueryReq(nodb, historyQuery)
req.SQLTemplate = mocks.NewTestingSQLTemplate()
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
require.NoError(t, err)
sql := rawQuery
// Verify that direct dashboard_version fields are used (no COALESCE)
require.Contains(t, sql, "dashboard_version.created as updated",
"Regular history SQL should use direct dashboard_version.created")
require.Contains(t, sql, "dashboard_version.version",
"Regular history SQL should use direct dashboard_version.version")
require.Contains(t, sql, "dashboard_version.data",
"Regular history SQL should use direct dashboard_version.data")
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
// to allow for cases where dashboard_version entries might be missing
// Should not contain COALESCE functions
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
"Regular history SQL should not contain COALESCE for updated")
})
}
func TestMigrateDashboardsConfiguration(t *testing.T) {
// Test the actual migration function configuration
t.Run("Migration options should configure query correctly", func(t *testing.T) {
// Test the migration configuration as used in real migration
opts := MigrateOptions{
WithHistory: true, // Migration includes history
}
// This simulates what happens in migrateDashboards function
expectedQuery := &DashboardQuery{
OrgID: 1,
Limit: 100000000,
GetHistory: opts.WithHistory, // Should be true
AllowFallback: true, // Should be true for migration
Order: "ASC", // Should be ASC for migration
}
// Verify the configuration matches what migration sets up
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
// Verify UseHistoryTable logic
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
})
}
+10 -2
View File
@@ -55,10 +55,9 @@ type LibraryPanelQuery struct {
LastID int64
}
type DashboardAccess interface {
type DashboardAccessor interface {
resource.StorageBackend
resourcepb.ResourceIndexServer
LegacyMigrator
GetDashboard(ctx context.Context, orgId int64, uid string, version int64) (*dashboardV1.Dashboard, int64, error)
SaveDashboard(ctx context.Context, orgId int64, dash *dashboardV1.Dashboard, failOnExisting bool) (*dashboardV1.Dashboard, bool, error)
@@ -67,3 +66,12 @@ type DashboardAccess interface {
// Get a typed list
GetLibraryPanels(ctx context.Context, query LibraryPanelQuery) (*dashboardV0.LibraryPanelList, error)
}
//go:generate mockery --name MigrationDashboardAccessor --structname MockMigrationDashboardAccessor --inpackage --filename migration_dashboard_accessor_mock.go --with-expecter
type MigrationDashboardAccessor interface {
// Migration helper methods - these support the separate LegacyMigrator
CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
}
@@ -22,7 +22,7 @@ import (
)
type DashboardStorage struct {
Access legacy.DashboardAccess
Access legacy.DashboardAccessor
DashboardService dashboards.DashboardService
}
+1 -1
View File
@@ -30,7 +30,7 @@ var (
)
type LibraryPanelStore struct {
Access legacy.DashboardAccess
Access legacy.DashboardAccessor
ResourceInfo utils.ResourceInfo
service libraryelements.Service
}
+1 -1
View File
@@ -169,7 +169,7 @@ func RegisterAPIService(
publicDashboardService: publicDashboardService,
legacy: &DashboardStorage{
Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
Access: legacy.NewDashboardSQLAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
DashboardService: dashboardService,
},
}
+2 -2
View File
@@ -30,7 +30,7 @@ type dtoBuilder = func(dashboard runtime.Object, access *dashboard.DashboardAcce
// The DTO returns everything the UI needs in a single request
type DTOConnector struct {
getter rest.Getter
legacy legacy.DashboardAccess
legacy legacy.DashboardAccessor
unified resource.ResourceClient
largeObjects apistore.LargeObjectSupport
accessControl accesscontrol.AccessControl
@@ -42,7 +42,7 @@ type DTOConnector struct {
func NewDTOConnector(
getter rest.Getter,
largeObjects apistore.LargeObjectSupport,
legacyAccess legacy.DashboardAccess,
legacyAccess legacy.DashboardAccessor,
resourceClient resource.ResourceClient,
accessControl accesscontrol.AccessControl,
scheme *runtime.Scheme,
@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources/signature"
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
"github.com/grafana/grafana/pkg/storage/unified/parquet"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
@@ -29,7 +30,7 @@ type LegacyResourcesMigrator interface {
type legacyResourcesMigrator struct {
repositoryResources resources.RepositoryResourcesFactory
parsers resources.ParserFactory
legacyMigrator legacy.LegacyMigrator
dashboardAccess legacy.MigrationDashboardAccessor
signerFactory signature.SignerFactory
clients resources.ClientFactory
exportFn export.ExportFn
@@ -38,7 +39,7 @@ type legacyResourcesMigrator struct {
func NewLegacyResourcesMigrator(
repositoryResources resources.RepositoryResourcesFactory,
parsers resources.ParserFactory,
legacyMigrator legacy.LegacyMigrator,
dashboardAccess legacy.MigrationDashboardAccessor,
signerFactory signature.SignerFactory,
clients resources.ClientFactory,
exportFn export.ExportFn,
@@ -46,7 +47,7 @@ func NewLegacyResourcesMigrator(
return &legacyResourcesMigrator{
repositoryResources: repositoryResources,
parsers: parsers,
legacyMigrator: legacyMigrator,
dashboardAccess: dashboardAccess,
signerFactory: signerFactory,
clients: clients,
exportFn: exportFn,
@@ -94,7 +95,7 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
reader := newLegacyResourceMigrator(
rw,
m.legacyMigrator,
m.dashboardAccess,
parser,
repositoryResources,
progress,
@@ -113,21 +114,21 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
}
type legacyResourceResourceMigrator struct {
repo repository.ReaderWriter
legacy legacy.LegacyMigrator
parser resources.Parser
progress jobs.JobProgressRecorder
namespace string
kind schema.GroupResource
options provisioning.MigrateJobOptions
resources resources.RepositoryResources
signer signature.Signer
history map[string]string // UID >> file path
repo repository.ReaderWriter
dashboardAccess legacy.MigrationDashboardAccessor
parser resources.Parser
progress jobs.JobProgressRecorder
namespace string
kind schema.GroupResource
options provisioning.MigrateJobOptions
resources resources.RepositoryResources
signer signature.Signer
history map[string]string // UID >> file path
}
func newLegacyResourceMigrator(
repo repository.ReaderWriter,
legacy legacy.LegacyMigrator,
dashboardAccess legacy.MigrationDashboardAccessor,
parser resources.Parser,
resources resources.RepositoryResources,
progress jobs.JobProgressRecorder,
@@ -141,16 +142,16 @@ func newLegacyResourceMigrator(
history = make(map[string]string)
}
return &legacyResourceResourceMigrator{
repo: repo,
legacy: legacy,
parser: parser,
progress: progress,
options: options,
namespace: namespace,
kind: kind,
resources: resources,
signer: signer,
history: history,
repo: repo,
dashboardAccess: dashboardAccess,
parser: parser,
progress: progress,
options: options,
namespace: namespace,
kind: kind,
resources: resources,
signer: signer,
history: history,
}
}
@@ -225,14 +226,21 @@ func (r *legacyResourceResourceMigrator) Write(ctx context.Context, key *resourc
func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
r.progress.SetMessage(ctx, fmt.Sprintf("migrate %s resource", r.kind.Resource))
// Create a parquet migrator with this instance as the BulkResourceWriter
parquetClient := parquet.NewBulkResourceWriterClient(r)
migrator := unifiedmigrations.ProvideUnifiedMigratorParquet(
r.dashboardAccess,
parquetClient,
)
opts := legacy.MigrateOptions{
Namespace: r.namespace,
WithHistory: r.options.History,
Resources: []schema.GroupResource{r.kind},
Store: parquet.NewBulkResourceWriterClient(r),
OnlyCount: true, // first get the count
}
stats, err := r.legacy.Migrate(ctx, opts)
stats, err := migrator.Migrate(ctx, opts)
if err != nil {
return fmt.Errorf("unable to count legacy items %w", err)
}
@@ -248,7 +256,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
}
opts.OnlyCount = false // this time actually write
_, err = r.legacy.Migrate(ctx, opts)
_, err = migrator.Migrate(ctx, opts)
if err != nil {
return fmt.Errorf("migrate legacy %s: %w", r.kind.Resource, err)
}
@@ -93,8 +93,8 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
mockRepoResourcesFactory.On("Client", mock.Anything, mock.Anything).
Return(mockRepoResources, nil)
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, errors.New("legacy migrator error"))
@@ -115,7 +115,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
migrator := NewLegacyResourcesMigrator(
mockRepoResourcesFactory,
mockParserFactory,
mockLegacyMigrator,
mockDashboardAccess,
signerFactory,
mockClientFactory,
mockExportFn.Execute,
@@ -136,7 +136,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
mockParserFactory.AssertExpectations(t)
mockRepoResourcesFactory.AssertExpectations(t)
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
mockExportFn.AssertExpectations(t)
mockClientFactory.AssertExpectations(t)
@@ -308,22 +308,18 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
History: true,
}).Return(mockSigner, nil)
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
// Mock CountResources for the count phase
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
})).Return(&resourcepb.BulkResponse{}, nil).Once()
// Mock MigrateDashboards for the actual migration phase (dashboards resource)
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return !opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{
Summary: []*resourcepb.BulkResponse_Summary{
{
Group: "test.grafana.app",
Resource: "tests",
Count: 10,
History: 5,
},
},
}, nil).Once() // Migration phase
}), mock.Anything).Return(&legacy.BlobStoreInfo{
Count: 10,
Size: 5,
}, nil).Once()
mockClients := resources.NewMockResourceClients(t)
mockClientFactory := resources.NewMockClientFactory(t)
@@ -339,7 +335,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
migrator := NewLegacyResourcesMigrator(
mockRepoResourcesFactory,
mockParserFactory,
mockLegacyMigrator,
mockDashboardAccess,
mockSignerFactory,
mockClientFactory,
mockExportFn.Execute,
@@ -362,7 +358,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
mockParserFactory.AssertExpectations(t)
mockRepoResourcesFactory.AssertExpectations(t)
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
mockClientFactory.AssertExpectations(t)
mockExportFn.AssertExpectations(t)
progress.AssertExpectations(t)
@@ -742,8 +738,8 @@ func TestLegacyResourceResourceMigrator_Write(t *testing.T) {
func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
t.Run("should fail when legacy migrate count fails", func(t *testing.T) {
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, errors.New("count error"))
@@ -752,7 +748,7 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
migrator := newLegacyResourceMigrator(
nil,
mockLegacyMigrator,
mockDashboardAccess,
nil,
nil,
progress,
@@ -766,89 +762,91 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
require.Error(t, err)
require.Contains(t, err.Error(), "unable to count legacy items")
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
})
t.Run("should fail when legacy migrate write fails", func(t *testing.T) {
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
// For test-resources GroupResource, we don't know which method it will call, but since it's not dashboards/folders/librarypanels,
// the Migrate will fail trying to map the resource type. Let's make it dashboards for this test.
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return !opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, errors.New("write error")).Once() // Write phase
}), mock.Anything).Return(nil, errors.New("write error")).Once() // Write phase
progress := jobs.NewMockJobProgressRecorder(t)
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
migrator := newLegacyResourceMigrator(
nil,
mockLegacyMigrator,
mockDashboardAccess,
nil,
nil,
progress,
provisioning.MigrateJobOptions{},
"test-namespace",
schema.GroupResource{Group: "test.grafana.app", Resource: "test-resources"},
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
signature.NewGrafanaSigner(),
)
err := migrator.Migrate(context.Background())
require.Error(t, err)
require.Contains(t, err.Error(), "migrate legacy test-resources: write error")
require.Contains(t, err.Error(), "migrate legacy dashboards: write error")
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
})
t.Run("should successfully migrate resource", func(t *testing.T) {
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return !opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
progress := jobs.NewMockJobProgressRecorder(t)
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
migrator := newLegacyResourceMigrator(
nil,
mockLegacyMigrator,
mockDashboardAccess,
nil,
nil,
progress,
provisioning.MigrateJobOptions{},
"test-namespace",
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
signature.NewGrafanaSigner(),
)
err := migrator.Migrate(context.Background())
require.NoError(t, err)
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
})
t.Run("should set total to history if history is greater than count", func(t *testing.T) {
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{
Summary: []*resourcepb.BulkResponse_Summary{
{
Group: "test.grafana.app",
Resource: "tests",
Group: "dashboard.grafana.app",
Resource: "dashboards",
Count: 1,
History: 100,
},
},
}, nil).Once() // Count phase
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return !opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
progress := jobs.NewMockJobProgressRecorder(t)
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
@@ -856,39 +854,39 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
migrator := newLegacyResourceMigrator(
nil,
mockLegacyMigrator,
mockDashboardAccess,
nil,
nil,
progress,
provisioning.MigrateJobOptions{},
"test-namespace",
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
signature.NewGrafanaSigner(),
)
err := migrator.Migrate(context.Background())
require.NoError(t, err)
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
})
t.Run("should set total to count if history is less than count", func(t *testing.T) {
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{
Summary: []*resourcepb.BulkResponse_Summary{
{
Group: "test.grafana.app",
Resource: "tests",
Group: "dashboard.grafana.app",
Resource: "dashboards",
Count: 200,
History: 1,
},
},
}, nil).Once() // Count phase
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
return !opts.OnlyCount && opts.Namespace == "test-namespace"
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
progress := jobs.NewMockJobProgressRecorder(t)
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
@@ -897,20 +895,20 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
migrator := newLegacyResourceMigrator(
nil,
mockLegacyMigrator,
mockDashboardAccess,
nil,
nil,
progress,
provisioning.MigrateJobOptions{},
"test-namespace",
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
signer,
)
err := migrator.Migrate(context.Background())
require.NoError(t, err)
mockLegacyMigrator.AssertExpectations(t)
mockDashboardAccess.AssertExpectations(t)
progress.AssertExpectations(t)
})
}
+11 -8
View File
@@ -60,6 +60,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
"github.com/grafana/grafana/pkg/storage/unified/migrations"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@@ -109,7 +110,7 @@ type APIBuilder struct {
jobHistoryConfig *JobHistoryConfig
jobHistoryLoki *jobs.LokiJobHistory
resourceLister resources.ResourceLister
legacyMigrator legacy.LegacyMigrator
dashboardAccess legacy.MigrationDashboardAccessor
storageStatus dualwrite.Service
unified resource.ResourceClient
repoFactory repository.Factory
@@ -135,7 +136,7 @@ func NewAPIBuilder(
features featuremgmt.FeatureToggles,
unified resource.ResourceClient,
configProvider apiserver.RestConfigProvider,
legacyMigrator legacy.LegacyMigrator,
dashboardAccess legacy.MigrationDashboardAccessor,
storageStatus dualwrite.Service,
usageStats usagestats.Service,
access authlib.AccessChecker,
@@ -158,6 +159,7 @@ func NewAPIBuilder(
clients = resources.NewClientFactory(configProvider)
}
parsers := resources.NewParserFactory(clients)
legacyMigrator := migrations.ProvideUnifiedMigrator(dashboardAccess, unified)
resourceLister := resources.NewResourceListerForMigrations(unified, legacyMigrator, storageStatus)
b := &APIBuilder{
@@ -170,7 +172,7 @@ func NewAPIBuilder(
parsers: parsers,
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
resourceLister: resourceLister,
legacyMigrator: legacyMigrator,
dashboardAccess: dashboardAccess,
storageStatus: storageStatus,
unified: unified,
access: access,
@@ -234,7 +236,7 @@ func RegisterAPIService(
client resource.ResourceClient, // implements resource.RepositoryClient
configProvider apiserver.RestConfigProvider,
access authlib.AccessClient,
legacyMigrator legacy.LegacyMigrator,
dashboardAccess legacy.MigrationDashboardAccessor,
storageStatus dualwrite.Service,
usageStats usagestats.Service,
tracer tracing.Tracer,
@@ -258,7 +260,7 @@ func RegisterAPIService(
features,
client,
configProvider,
legacyMigrator, storageStatus,
dashboardAccess, storageStatus,
usageStats,
access,
tracer,
@@ -722,7 +724,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
legacyResources := migrate.NewLegacyResourcesMigrator(
b.repositoryResources,
b.parsers,
b.legacyMigrator,
b.dashboardAccess,
signerFactory,
b.clients,
export.ExportAll,
@@ -1241,8 +1243,9 @@ func (b *APIBuilder) tryRunningOnlyUnifiedStorage() error {
return nil
}
// Count how many things exist
rsp, err := b.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
// Count how many things exist - create a migrator on-demand for this
legacyMigrator := migrations.ProvideUnifiedMigrator(b.dashboardAccess, b.unified)
rsp, err := legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
Namespace: "default", // FIXME! this works for single org, but need to check multi-org
Resources: []schema.GroupResource{{
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
@@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
"github.com/grafana/grafana/pkg/storage/unified/migrations"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@@ -30,9 +31,9 @@ type ResourceStore interface {
}
type ResourceListerFromSearch struct {
store ResourceStore
legacyMigrator legacy.LegacyMigrator
storageStatus dualwrite.Service
store ResourceStore
migrator migrations.UnifiedMigrator
storageStatus dualwrite.Service
}
func NewResourceLister(store ResourceStore) ResourceLister {
@@ -42,13 +43,13 @@ func NewResourceLister(store ResourceStore) ResourceLister {
// FIXME: the logic about migration and storage should probably be separated from this
func NewResourceListerForMigrations(
store ResourceStore,
legacyMigrator legacy.LegacyMigrator,
migrator migrations.UnifiedMigrator,
storageStatus dualwrite.Service,
) ResourceLister {
return &ResourceListerFromSearch{
store: store,
legacyMigrator: legacyMigrator,
storageStatus: storageStatus,
store: store,
migrator: migrator,
storageStatus: storageStatus,
}
}
@@ -133,8 +134,8 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit
}
// Get the stats based on what a migration could support
if o.storageStatus != nil && o.legacyMigrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
if o.storageStatus != nil && o.migrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
rsp, err := o.migrator.Migrate(ctx, legacy.MigrateOptions{
Namespace: namespace,
Resources: []schema.GroupResource{{
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
@@ -48,7 +48,6 @@ import (
"github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl"
"github.com/grafana/grafana/pkg/services/team/teamapi"
"github.com/grafana/grafana/pkg/services/updatemanager"
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
)
func ProvideBackgroundServiceRegistry(
@@ -74,7 +73,6 @@ func ProvideBackgroundServiceRegistry(
dashboardServiceImpl *service.DashboardServiceImpl,
secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker,
fixedRolesLoader *accesscontrol.FixedRolesLoader,
unifiedStorageMigrationProvider unifiedmigrations.UnifiedStorageMigrationProvider,
// Need to make sure these are initialized, is there a better place to put them?
_ dashboardsnapshots.Service,
_ serviceaccounts.Service,
@@ -91,7 +89,6 @@ func ProvideBackgroundServiceRegistry(
notifications,
rendering,
tokenService,
unifiedStorageMigrationProvider,
provisioning,
grafanaUpdateChecker,
pluginsUpdateChecker,
+5 -3
View File
@@ -128,6 +128,7 @@ import (
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginaccesscontrol"
"github.com/grafana/grafana/pkg/services/preference/prefimpl"
promTypeMigration "github.com/grafana/grafana/pkg/services/promtypemigration"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/publicdashboards"
publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api"
publicdashboardsStore "github.com/grafana/grafana/pkg/services/publicdashboards/database"
@@ -238,7 +239,9 @@ var wireBasicSet = wire.NewSet(
uss.ProvideService,
wire.Bind(new(usagestats.Service), new(*uss.UsageStats)),
validator.ProvideService,
legacy.ProvideLegacyMigrator,
provisioning.ProvideStubProvisioningService,
legacy.ProvideMigratorDashboardAccessor,
unifiedmigrations.ProvideUnifiedMigrator,
pluginsintegration.WireSet,
pluginDashboards.ProvideFileStoreManager,
wire.Bind(new(pluginDashboards.FileStore), new(*pluginDashboards.FileStoreManager)),
@@ -466,8 +469,7 @@ var wireBasicSet = wire.NewSet(
// Unified storage
resource.ProvideStorageMetrics,
resource.ProvideIndexMetrics,
unifiedmigrations.ProvideUnifiedStorageMigrationProvider,
wire.Bind(new(unifiedmigrations.UnifiedStorageMigrationProvider), new(*unifiedmigrations.UnifiedStorageMigrationProviderImpl)),
unifiedmigrations.ProvideUnifiedStorageMigrationService,
// Kubernetes API server
grafanaapiserver.WireSet,
apiregistry.WireSet,
+30 -17
View File
File diff suppressed because one or more lines are too long
+2
View File
@@ -62,6 +62,7 @@ import (
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/services/validations"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/legacysql"
"github.com/grafana/grafana/pkg/storage/unified"
"github.com/grafana/grafana/pkg/storage/unified/resource"
search2 "github.com/grafana/grafana/pkg/storage/unified/search"
@@ -98,6 +99,7 @@ var wireExtsBasicSet = wire.NewSet(
wire.Bind(new(validations.DataSourceRequestURLValidator), new(*validations.OSSDataSourceRequestURLValidator)),
provisioning.ProvideService,
wire.Bind(new(provisioning.ProvisioningService), new(*provisioning.ProvisioningServiceImpl)),
legacysql.NewDatabaseProvider,
backgroundsvcs.ProvideBackgroundServiceRegistry,
wire.Bind(new(registry.BackgroundServiceRegistry), new(*backgroundsvcs.BackgroundServiceRegistry)),
migrations.ProvideOSSMigrations,
@@ -107,7 +107,7 @@ func NewDualWriter(
if currentMode != mode {
klog.Warningf("Requested DualWrite mode: %d, but using %d for %+v", mode, currentMode, gr)
}
return dualwrite.NewDualWriter(gr, currentMode, legacy, storage)
return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage)
}
func getRequestInfo(gr schema.GroupResource, namespaceMapper request.NamespaceMapper) *k8srequest.RequestInfo {
+1 -1
View File
@@ -379,7 +379,7 @@ func InstallAPIs(
case grafanarest.Mode4, grafanarest.Mode5:
return storage, nil
default:
return dualwrite.NewDualWriter(gr, currentMode, legacy, storage)
return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage)
}
}
}
+11 -6
View File
@@ -97,7 +97,7 @@ type service struct {
authorizer *authorizer.GrafanaAuthorizer
serverLockService builder.ServerLockService
storageStatus dualwrite.Service
dualWriter dualwrite.Service
kvStore kvstore.KVStore
pluginClient plugins.Client
@@ -127,7 +127,7 @@ func ProvideService(
datasources datasource.ScopedPluginDatasourceProvider,
contextProvider datasource.PluginContextWrapper,
pluginStore pluginstore.Store,
storageStatus dualwrite.Service,
dualWriter dualwrite.Service,
unified resource.ResourceClient,
secrets secret.InlineSecureValueSupport,
restConfigProvider RestConfigProvider,
@@ -158,7 +158,7 @@ func ProvideService(
contextProvider: contextProvider,
pluginStore: pluginStore,
serverLockService: serverLockService,
storageStatus: storageStatus,
dualWriter: dualWriter,
unified: unified,
secrets: secrets,
restConfigProvider: restConfigProvider,
@@ -385,12 +385,17 @@ func (s *service) start(ctx context.Context) error {
}
// Install the API group+version for existing builders
err = builder.InstallAPIs(s.scheme, s.codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions,
err = builder.InstallAPIs(s.scheme,
s.codecs,
server,
serverConfig.RESTOptionsGetter,
builders,
o.StorageOptions,
s.metrics,
request.GetNamespaceMapper(s.cfg),
kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
s.serverLockService,
s.storageStatus,
s.dualWriter,
optsregister,
s.features,
s.dualWriterMetrics,
@@ -409,7 +414,7 @@ func (s *service) start(ctx context.Context) error {
kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
s.serverLockService,
request.GetNamespaceMapper(s.cfg),
s.storageStatus,
s.dualWriter,
s.dualWriterMetrics,
s.builderMetrics,
serverConfig.MergedResourceConfig,
+69
View File
@@ -0,0 +1,69 @@
package provisioning
import (
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
"github.com/grafana/grafana/pkg/setting"
)
type StubProvisioningService interface {
GetDashboardProvisionerResolvedPath(name string) string
GetAllowUIUpdatesFromConfig(name string) bool
}
func ProvideStubProvisioningService(cfg *setting.Cfg) (StubProvisioningService, error) {
return NewStubProvisioning(cfg.ProvisioningPath)
}
func NewStubProvisioning(path string) (StubProvisioningService, error) {
cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
if err != nil {
return nil, err
}
stub := &stubProvisioning{
path: make(map[string]string),
allowUIUpdates: make(map[string]bool),
log: log.New("provisioning.stub"),
}
for _, cfg := range cfgs {
stub.path[cfg.Name] = cfg.Options["path"].(string)
stub.allowUIUpdates[cfg.Name] = cfg.AllowUIUpdates
}
return stub, nil
}
type stubProvisioning struct {
path map[string]string // name > options.path
allowUIUpdates map[string]bool
log log.Logger
}
func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
return s.allowUIUpdates[name]
}
func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
path := s.path[name]
if _, err := os.Stat(path); os.IsNotExist(err) {
s.log.Warn("Cannot read directory", "error", err)
}
path, err := filepath.Abs(path)
if err != nil {
s.log.Warn("Could not create absolute path", "path", path, "error", err)
}
path, err = filepath.EvalSymlinks(path)
if err != nil {
s.log.Warn("Failed to read content of symlinked path", "path", path, "error", err)
}
if path == "" {
path = s.path[name]
s.log.Info("falling back to original path due to EvalSymlink/Abs failure")
}
return path
}
@@ -71,7 +71,7 @@ func TestMode1_Create(t *testing.T) {
tt.setupStorageFn(us.Mock)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
@@ -154,7 +154,7 @@ func TestMode1_Get(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{})
@@ -217,7 +217,7 @@ func TestMode1_List(t *testing.T) {
tt.setupStorageFn(us.Mock)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
_, err = dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -286,7 +286,7 @@ func TestMode1_Delete(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
obj, _, err := dw.Delete(context.Background(), name, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -361,7 +361,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
tt.setupStorageFn(us.Mock, tt.input)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
@@ -434,7 +434,7 @@ func TestMode1_Update(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
require.NoError(t, err)
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
@@ -69,7 +69,7 @@ func TestMode2_Create(t *testing.T) {
tt.setupStorageFn(us.Mock, tt.input)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
@@ -154,7 +154,7 @@ func TestMode2_Get(t *testing.T) {
tt.setupStorageFn(us.Mock, tt.input)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
@@ -229,7 +229,7 @@ func TestMode2_List(t *testing.T) {
tt.setupStorageFn(us.Mock)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -331,7 +331,7 @@ func TestMode2_Delete(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -401,7 +401,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
tt.setupStorageFn(us.Mock)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{})
@@ -471,7 +471,7 @@ func TestMode2_Update(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
require.NoError(t, err)
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
@@ -75,7 +75,7 @@ func TestMode3_Create(t *testing.T) {
tt.setupStorageFn(us.Mock, tt.input)
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
@@ -134,7 +134,7 @@ func TestMode3_Get(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{})
@@ -185,7 +185,7 @@ func TestMode3_List(t *testing.T) {
tt.setupStorageFn(us.Mock, &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}})
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
res, err := dw.List(context.Background(), &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}})
@@ -276,7 +276,7 @@ func TestMode3_Delete(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -346,7 +346,7 @@ func TestMode3_DeleteCollection(t *testing.T) {
tt.setupStorageFn(us.Mock)
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{})
@@ -416,7 +416,7 @@ func TestMode3_Update(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
require.NoError(t, err)
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
@@ -14,37 +14,6 @@ import (
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
)
func (m *service) NewStorage(gr schema.GroupResource, legacy grafanarest.Storage, unified grafanarest.Storage) (grafanarest.Storage, error) {
status, err := m.Status(context.Background(), gr)
if err != nil {
return nil, err
}
if m.enabled && status.Runtime {
// Dynamic storage behavior
return &runtimeDualWriter{
service: m,
legacy: legacy,
unified: unified,
dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read
gr: gr,
}, nil
}
if status.ReadUnified {
if status.WriteLegacy {
// Write both, read unified
return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil
}
return unified, nil
}
if status.WriteUnified {
// Write both, read legacy
return &dualWriter{legacy: legacy, unified: unified}, nil
}
return legacy, nil
}
// The runtime dual writer implements the various modes we have described as: mode:1/2/3/4/5
// However the behavior can be configured at runtime rather than just at startup.
// When a resource is marked as "migrating", all write requests will be 503 unavailable
@@ -77,7 +77,7 @@ func TestRuntime_Create(t *testing.T) {
tt.setupStorageFn(us.Mock, tt.input)
}
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
require.NoError(t, err)
dw, err := m.NewStorage(kind, ls, us)
require.NoError(t, err)
@@ -150,7 +150,7 @@ func TestRuntime_Get(t *testing.T) {
tt.setupStorageFn(us.Mock, name)
}
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
require.NoError(t, err)
dw, err := m.NewStorage(kind, ls, us)
require.NoError(t, err)
@@ -235,7 +235,7 @@ func TestRuntime_CreateWhileMigrating(t *testing.T) {
}
// Shared provider across all tests
dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
require.NoError(t, err)
for _, tt := range tests {
@@ -12,8 +12,28 @@ import (
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
)
// fakeMigrator is a no-op implementation of UnifiedStorageMigrationService
type fakeMigrator struct{}
func (f *fakeMigrator) Run(ctx context.Context) error {
return nil
}
var _ unifiedmigrations.UnifiedStorageMigrationService = (*fakeMigrator)(nil)
func NewFakeMigrator() unifiedmigrations.UnifiedStorageMigrationService {
return &fakeMigrator{}
}
func NewFakeConfig() *setting.Cfg {
return &setting.Cfg{
UnifiedStorage: make(map[string]setting.UnifiedStorageConfig),
}
}
func ProvideStaticServiceForTests(cfg *setting.Cfg) Service {
if cfg == nil {
cfg = &setting.Cfg{}
@@ -25,7 +45,14 @@ func ProvideService(
features featuremgmt.FeatureToggles,
kv kvstore.KVStore,
cfg *setting.Cfg,
migrator unifiedmigrations.UnifiedStorageMigrationService,
) (Service, error) {
// Ensure migrations have run before starting dualwrite
err := migrator.Run(context.Background())
if err != nil {
return nil, fmt.Errorf("unable to start dualwrite service due to migration error: %w", err)
}
//nolint:staticcheck // not yet migrated to OpenFeature
enabled := features.IsEnabledGlobally(featuremgmt.FlagManagedDualWriter) ||
features.IsEnabledGlobally(featuremgmt.FlagProvisioning) // required for git provisioning
@@ -64,6 +91,37 @@ type service struct {
enabled bool
}
func (m *service) NewStorage(gr schema.GroupResource, legacy rest.Storage, unified rest.Storage) (rest.Storage, error) {
status, err := m.Status(context.Background(), gr)
if err != nil {
return nil, err
}
if m.enabled && status.Runtime {
// Dynamic storage behavior
return &runtimeDualWriter{
service: m,
legacy: legacy,
unified: unified,
dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read
gr: gr,
}, nil
}
if status.ReadUnified {
if status.WriteLegacy {
// Write both, read unified
return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil
}
return unified, nil
}
if status.WriteUnified {
// Write both, read legacy
return &dualWriter{legacy: legacy, unified: unified}, nil
}
return legacy, nil
}
// Hardcoded list of resources that should be controlled by the database (eventually everything?)
func (m *service) ShouldManage(gr schema.GroupResource) bool {
if !m.enabled {
@@ -17,7 +17,7 @@ import (
func TestService(t *testing.T) {
t.Run("dynamic", func(t *testing.T) {
ctx := context.Background()
mode, err := ProvideService(featuremgmt.WithFeatures(), kvstore.NewFakeKVStore(), nil)
mode, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagProvisioning), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
require.NoError(t, err)
gr := schema.GroupResource{Group: "ggg", Resource: "rrr"}
@@ -122,7 +122,7 @@ func TestService(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg)
svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg, NewFakeMigrator())
if tc.error != "" {
require.ErrorContains(t, err, tc.error)
require.Nil(t, svc, "expect a nil service when an error exts")
+2 -2
View File
@@ -10,8 +10,8 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
// NewDualWriter -- temporary shim
func NewDualWriter(
// NewStaticStorage -- temporary shim
func NewStaticStorage(
gr schema.GroupResource,
mode rest.DualWriterMode,
legacy rest.Storage,
@@ -0,0 +1,12 @@
package contract
import (
"github.com/grafana/grafana/pkg/registry"
)
// UnifiedStorageMigrationService provides unified storage migrations as a background service.
// This interface is defined in a separate package to avoid import cycles between
// the migrations implementation and packages that need to depend on it (like dualwrite).
type UnifiedStorageMigrationService interface {
registry.BackgroundService
}
@@ -1,93 +0,0 @@
package migrations
import (
"context"
"fmt"
"github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/util/xorm"
"k8s.io/apimachinery/pkg/runtime/schema"
)
const (
FoldersAndDashboardsMigrationID = "folders and dashboards migration"
UnifiedStorageDataMigrationSQL = "unified storage data migration"
)
type dashboardAndFolderMigration struct {
migrator.MigrationBase
legacyMigrator legacy.LegacyMigrator
bulkStoreClient resource.ResourceClient
}
var _ migrator.CodeMigration = (*dashboardAndFolderMigration)(nil)
// SQL implements migrator.Migration interface. Returns a description string.
func (sp *dashboardAndFolderMigration) SQL(dialect migrator.Dialect) string {
return UnifiedStorageDataMigrationSQL
}
func (sp *dashboardAndFolderMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
ctx := context.Background()
logger := mg.Logger
resources := []schema.GroupResource{
{
Group: "folder.grafana.app",
Resource: "folders",
},
{
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
}
storageMigrator := newUnifiedStorageMigrator(sp.legacyMigrator, sp.bulkStoreClient, resources, "unified-storage-migration.folders-dashboards")
orgs, err := sp.getAllOrgs(sess)
if err != nil {
logger.Error("failed to get organizations for folders and dashboards migration", "error", err)
return fmt.Errorf("failed to get organizations: %w", err)
}
if len(orgs) == 0 {
logger.Info("No organizations found to migrate, skipping migration")
return nil
}
logger.Info("Starting migration for all organizations", "org_count", len(orgs))
for _, org := range orgs {
namespace := types.OrgNamespaceFormatter(org.ID)
logger.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
// Create a service identity context for this namespace to authenticate with unified storage
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
if err := storageMigrator.executeMigration(migrationCtx, sess, mg, namespace); err != nil {
logger.Error("migration failed for organization", "org_id", org.ID, "org_name", org.Name, "error", err)
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
}
}
logger.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
return nil
}
type orgInfo struct {
ID int64 `xorm:"id"`
Name string `xorm:"name"`
}
func (sp *dashboardAndFolderMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
var orgs []orgInfo
err := sess.Table("org").Cols("id", "name").Find(&orgs)
if err != nil {
return nil, err
}
return orgs, nil
}
@@ -1,112 +0,0 @@
package migrations
import (
"context"
"fmt"
"os"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel"
)
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
var logger = log.New("storage.unified.migrations")
// UnifiedStorageMigrationProvider provides unified storage migrations as a background service
type UnifiedStorageMigrationProvider interface {
registry.BackgroundService
}
type UnifiedStorageMigrationProviderImpl struct {
legacyMigrator legacy.LegacyMigrator
cfg *setting.Cfg
client resource.ResourceClient
sqlStore db.DB
}
var _ UnifiedStorageMigrationProvider = (*UnifiedStorageMigrationProviderImpl)(nil)
// ProvideUnifiedStorageMigrationProvider is a Wire provider that creates the migration service.
// The service implements registry.BackgroundService and runs migrations during server startup.
func ProvideUnifiedStorageMigrationProvider(
legacyMigrator legacy.LegacyMigrator,
cfg *setting.Cfg,
client resource.ResourceClient,
sqlStore db.DB,
) *UnifiedStorageMigrationProviderImpl {
return &UnifiedStorageMigrationProviderImpl{
legacyMigrator: legacyMigrator,
cfg: cfg,
client: client,
sqlStore: sqlStore,
}
}
// Run executes unified storage migrations as a background service.
// This blocks until migrations complete. If migrations fail, an error is returned
// which will prevent Grafana from starting.
func (p *UnifiedStorageMigrationProviderImpl) Run(ctx context.Context) error {
// skip migrations in test environments to prevent integration test timeouts.
if os.Getenv("GRAFANA_TEST_DB") != "" {
return nil
}
// skip migrations if disabled in config
if p.cfg.DisableDataMigrations {
logger.Info("Data migrations are disabled, skipping")
return nil
}
// TODO: Re-enable once migrations are ready
// return RegisterMigrations(p.legacyMigrator, p.cfg, p.client, p.sqlStore)
return nil
}
// RegisterMigrations initializes and registers all unified storage migrations.
// This function is the entry point for all data migrations from legacy storage
// to unified storage. It returns an error if migrations fail, preventing Grafana
// from starting with inconsistent data.
func RegisterMigrations(
legacyMigrator legacy.LegacyMigrator,
cfg *setting.Cfg,
client resource.ResourceClient,
sqlStore db.DB,
) error {
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
defer span.End()
mg := migrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unified_storage")
mg.AddCreateMigration()
if err := prometheus.Register(mg); err != nil {
logger.Warn("Failed to register migrator metrics", "error", err)
}
// Add new migration registrations here for each resource type
registerDashboardAndFolderMigration(mg, legacyMigrator, client)
// Run all registered migrations (blocking)
sec := cfg.Raw.Section("database")
if err := mg.RunMigrations(ctx, sec.Key("migration_locking").MustBool(true), sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
return fmt.Errorf("unified storage data migration failed: %w", err)
}
logger.Info("Unified storage migrations completed successfully")
return nil
}
func registerDashboardAndFolderMigration(
mg *migrator.Migrator,
legacyMigrator legacy.LegacyMigrator,
bulkStoreClient resource.ResourceClient,
) {
migration := &dashboardAndFolderMigration{
legacyMigrator: legacyMigrator,
bulkStoreClient: bulkStoreClient,
}
mg.AddMigration(FoldersAndDashboardsMigrationID, migration)
}
+174 -138
View File
@@ -3,164 +3,200 @@ package migrations
import (
"context"
"fmt"
"time"
"github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"google.golang.org/grpc/metadata"
authlib "github.com/grafana/authlib/types"
v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/util/xorm"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// StorageMigrator defines the interface for executing unified storage migrations
type StorageMigrator interface {
executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error
// Read from legacy and write into unified storage
//
//go:generate mockery --name UnifiedMigrator --structname MockUnifiedMigrator --inpackage --filename migrator_mock.go --with-expecter
type UnifiedMigrator interface {
Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error)
}
type unifiedStorageMigrator struct {
migrator legacy.LegacyMigrator
bulkStoreClient resource.ResourceClient
resources []schema.GroupResource
log log.Logger
// unifiedMigration handles the migration of legacy resources to unified storage
type unifiedMigration struct {
legacy.MigrationDashboardAccessor
streamProvider streamProvider
log log.Logger
}
func newUnifiedStorageMigrator(migrator legacy.LegacyMigrator, bulkStoreClient resource.ResourceClient, resources []schema.GroupResource, logPrefix string) StorageMigrator {
return &unifiedStorageMigrator{
migrator: migrator,
bulkStoreClient: bulkStoreClient,
resources: resources,
log: log.New(logPrefix),
}
// streamProvider abstracts the different ways to create a bulk process stream
type streamProvider interface {
createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error)
}
func (m *unifiedStorageMigrator) executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error {
startTime := time.Now()
m.log.Info("Starting unified storage migration", "namespace", namespace, "resources", m.resources)
opts := legacy.MigrateOptions{
Namespace: namespace,
Store: m.bulkStoreClient,
LargeObjects: nil, // Not using large object support to avoid import cycles
Resources: m.resources,
WithHistory: true, // Migrate with full history
OnlyCount: false,
Progress: func(count int, msg string) {
m.log.Info("Migration progress", "count", count, "message", msg)
},
}
// Execute the migration via legacy migrator
response, err := m.migrator.Migrate(ctx, opts)
if err != nil {
m.log.Error("Migration failed", "error", err, "duration", time.Since(startTime))
return fmt.Errorf("failed to migrate resources: %w", err)
}
// Validate the migration results
if err := m.validateMigration(sess, response); err != nil {
m.log.Error("Migration validation failed", "error", err, "duration", time.Since(startTime))
return fmt.Errorf("migration validation failed: %w", err)
}
m.log.Info("Migration completed successfully",
"duration", time.Since(startTime),
"processed", response.Processed,
"summaries", len(response.Summary),
"rejected", len(response.Rejected))
return nil
// resourceClientStreamProvider creates streams using resource.ResourceClient
type resourceClientStreamProvider struct {
client resource.ResourceClient
}
func (m *unifiedStorageMigrator) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
// Check for rejected items
if len(response.Rejected) > 0 {
m.log.Warn("Migration had rejected items", "count", len(response.Rejected))
for i, rejected := range response.Rejected {
if i < 10 { // Log first 10 rejected items
m.log.Warn("Rejected item",
"namespace", rejected.Key.Namespace,
"group", rejected.Key.Group,
"resource", rejected.Key.Resource,
"name", rejected.Key.Name,
"reason", rejected.Error)
}
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
// Build collection settings for resource client
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
Resource: folders.RESOURCE,
})
case "dashboard.grafana.app/librarypanels":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
})
case "dashboard.grafana.app/dashboards":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.DASHBOARD_RESOURCE,
})
}
// Rejections are not fatal - they may be expected for invalid data
}
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
return r.client.BulkProcess(ctx)
}
// bulkStoreClientStreamProvider creates streams using resourcepb.BulkStoreClient
type bulkStoreClientStreamProvider struct {
client resourcepb.BulkStoreClient
}
func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
// Build collection settings for resource client
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
Resource: folders.RESOURCE,
})
case "dashboard.grafana.app/librarypanels":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
})
case "dashboard.grafana.app/dashboards":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.DASHBOARD_RESOURCE,
})
}
}
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
return b.client.BulkProcess(ctx)
}
// This can migrate Folders, Dashboards and LibraryPanels
func ProvideUnifiedMigrator(
dashboardAccess legacy.MigrationDashboardAccessor,
client resource.ResourceClient,
) UnifiedMigrator {
return newUnifiedMigrator(
dashboardAccess,
&resourceClientStreamProvider{client: client},
log.New("storage.unified.migrator"),
)
}
func ProvideUnifiedMigratorParquet(
dashboardAccess legacy.MigrationDashboardAccessor,
client resourcepb.BulkStoreClient,
) UnifiedMigrator {
return newUnifiedMigrator(
dashboardAccess,
&bulkStoreClientStreamProvider{client: client},
log.New("storage.unified.migrator.parquet"),
)
}
func newUnifiedMigrator(
dashboardAccess legacy.MigrationDashboardAccessor,
streamProvider streamProvider,
log log.Logger,
) UnifiedMigrator {
return &unifiedMigration{
MigrationDashboardAccessor: dashboardAccess,
streamProvider: streamProvider,
log: log,
}
}
// migrate function -- works for a single kind
type migratorFunc = func(ctx context.Context, orgId int64, opts legacy.MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*legacy.BlobStoreInfo, error)
func (m *unifiedMigration) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
info, err := authlib.ParseNamespace(opts.Namespace)
if err != nil {
return nil, err
}
if opts.Progress == nil {
opts.Progress = func(count int, msg string) {} // noop
}
// Validate counts for each resource type
for _, summary := range response.Summary {
legacyCount, err := m.getLegacyCount(sess, summary.Group, summary.Resource, summary.Namespace)
if len(opts.Resources) < 1 {
return nil, fmt.Errorf("missing resource selector")
}
if opts.OnlyCount {
return m.CountResources(ctx, opts)
}
stream, err := m.streamProvider.createStream(ctx, opts)
if err != nil {
return nil, err
}
migratorFuncs := []migratorFunc{}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
migratorFuncs = append(migratorFuncs, m.MigrateFolders)
case "dashboard.grafana.app/librarypanels":
migratorFuncs = append(migratorFuncs, m.MigrateLibraryPanels)
case "dashboard.grafana.app/dashboards":
migratorFuncs = append(migratorFuncs, m.MigrateDashboards)
default:
return nil, fmt.Errorf("unsupported resource: %s", res)
}
}
// Execute migrations
blobStore := legacy.BlobStoreInfo{}
m.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
for _, fn := range migratorFuncs {
blobs, err := fn(ctx, info.OrgID, opts, stream)
if err != nil {
return fmt.Errorf("failed to get legacy count for %s/%s: %w", summary.Group, summary.Resource, err)
m.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
return nil, err
}
// Account for rejected items in validation
expectedCount := summary.Count + int64(len(response.Rejected))
m.log.Info("Count validation",
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
"namespace", summary.Namespace,
"legacy_count", legacyCount,
"unified_count", summary.Count,
"rejected", len(response.Rejected),
"history", summary.History)
// Validate that we migrated all items (allowing for rejected items)
if legacyCount > expectedCount {
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
summary.Resource, summary.Group, summary.Namespace,
legacyCount, summary.Count, len(response.Rejected))
if blobs != nil {
blobStore.Count += blobs.Count
blobStore.Size += blobs.Size
}
}
return nil
}
func (m *unifiedStorageMigrator) getLegacyCount(sess *xorm.Session, group, resourceType, namespace string) (int64, error) {
// Parse namespace to get org ID
orgID, err := ParseOrgIDFromNamespace(namespace)
if err != nil {
return 0, fmt.Errorf("invalid namespace %s: %w", namespace, err)
}
// Map group/resource to legacy table
tableName, whereClause := m.getLegacyTableInfo(group, resourceType)
if tableName == "" {
return 0, fmt.Errorf("unknown resource type: %s.%s", resourceType, group)
}
// Count items in legacy table using Table() before Count()
count, err := sess.Table(tableName).Where(whereClause, orgID).Count()
if err != nil {
return 0, fmt.Errorf("failed to count %s: %w", tableName, err)
}
return count, nil
}
func (m *unifiedStorageMigrator) getLegacyTableInfo(group, resource string) (table string, whereClause string) {
// Map unified storage group/resource to legacy tables
switch {
case group == "dashboard.grafana.app" && resource == "dashboards":
return "dashboard", "org_id = ? and is_folder = false"
case group == "folder.grafana.app" && resource == "folders":
return "dashboard", "org_id = ? and is_folder = true"
case group == "playlist.grafana.app" && resource == "playlists":
return "playlist", "org_id = ?"
default:
return "", ""
}
}
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
// Use authlib to properly parse all namespace formats including "default" for org 1
info, err := types.ParseNamespace(namespace)
if err != nil {
return 0, fmt.Errorf("failed to parse namespace: %w", err)
}
return info.OrgID, nil
m.log.Info("finished migrating legacy resources", "blobStore", blobStore)
return stream.CloseAndRecv()
}
@@ -0,0 +1,98 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
package migrations
import (
context "context"
legacy "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
mock "github.com/stretchr/testify/mock"
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
// MockUnifiedMigrator is an autogenerated mock type for the UnifiedMigrator type
type MockUnifiedMigrator struct {
mock.Mock
}
type MockUnifiedMigrator_Expecter struct {
mock *mock.Mock
}
func (_m *MockUnifiedMigrator) EXPECT() *MockUnifiedMigrator_Expecter {
return &MockUnifiedMigrator_Expecter{mock: &_m.Mock}
}
// Migrate provides a mock function with given fields: ctx, opts
func (_m *MockUnifiedMigrator) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
ret := _m.Called(ctx, opts)
if len(ret) == 0 {
panic("no return value specified for Migrate")
}
var r0 *resourcepb.BulkResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
return rf(ctx, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) *resourcepb.BulkResponse); ok {
r0 = rf(ctx, opts)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*resourcepb.BulkResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, legacy.MigrateOptions) error); ok {
r1 = rf(ctx, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockUnifiedMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
type MockUnifiedMigrator_Migrate_Call struct {
*mock.Call
}
// Migrate is a helper method to define mock.On call
// - ctx context.Context
// - opts legacy.MigrateOptions
func (_e *MockUnifiedMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockUnifiedMigrator_Migrate_Call {
return &MockUnifiedMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
}
func (_c *MockUnifiedMigrator_Migrate_Call) Run(run func(ctx context.Context, opts legacy.MigrateOptions)) *MockUnifiedMigrator_Migrate_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(legacy.MigrateOptions))
})
return _c
}
func (_c *MockUnifiedMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockUnifiedMigrator_Migrate_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockUnifiedMigrator_Migrate_Call) RunAndReturn(run func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)) *MockUnifiedMigrator_Migrate_Call {
_c.Call.Return(run)
return _c
}
// NewMockUnifiedMigrator creates a new instance of MockUnifiedMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockUnifiedMigrator(t interface {
mock.TestingT
Cleanup(func())
}) *MockUnifiedMigrator {
mock := &MockUnifiedMigrator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -0,0 +1,275 @@
package migrations
import (
"context"
"fmt"
"time"
"github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/util/xorm"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// ValidationFunc is a function that validates migration results.
// It receives the database session, migration response, and logger for reporting.
// Return an error if validation fails, nil if validation passes or is skipped.
type ValidationFunc func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error
// ResourceMigration handles migration of specific resource types from legacy to unified storage.
// It implements migrator.CodeMigration and provides a generic, extensible way to migrate any
// resource type by:
//
// 1. Iterating through all organizations
// 2. For each org, delegating to LegacyMigrator to read from legacy and write to unified storage
// 3. Validating migration results using the provided validation function (if any)
//
// To add a new resource type migration, simply create a new ResourceMigration instance in
// service.go with the appropriate schema.GroupResource specifications and optional validation function.
type ResourceMigration struct {
migrator.MigrationBase
migrator UnifiedMigrator
resources []schema.GroupResource
migrationID string
validationFunc ValidationFunc // Optional: custom validation logic for this migration
log log.Logger
}
// NewResourceMigration creates a new migration for the specified resources.
// This is the primary way to register new resource migrations.
//
// Parameters:
// - legacyMigrator: handles reading from legacy storage and writing to unified storage
// - resources: list of GroupResource to migrate
// - migrationID: unique identifier for this migration
// - validationFunc: optional validation function to verify migration results.
// If nil, no validation will be performed.
//
// Example with legacy table count validation:
//
// NewResourceMigration(
// migrator,
// []schema.GroupResource{{Group: "playlist.grafana.app", Resource: "playlists"}},
// "playlists",
// NewLegacyTableCountValidator(map[string]LegacyTableInfo{
// "playlist.grafana.app/playlists": {Table: "playlist", WhereClause: "org_id = ?"},
// }),
// )
//
// Example without validation:
//
// NewResourceMigration(migrator, resources, "new-resource", nil)
func NewResourceMigration(
migrator UnifiedMigrator,
resources []schema.GroupResource,
migrationID string,
validationFunc ValidationFunc,
) *ResourceMigration {
return &ResourceMigration{
migrator: migrator,
resources: resources,
migrationID: migrationID,
validationFunc: validationFunc,
log: log.New("storage.unified.resource_migration." + migrationID),
}
}
var _ migrator.CodeMigration = (*ResourceMigration)(nil)
// SQL implements migrator.Migration interface. Returns a description string.
func (m *ResourceMigration) SQL(_ migrator.Dialect) string {
return fmt.Sprintf("unified storage data migration: %s", m.migrationID)
}
// Exec implements migrator.CodeMigration interface. Executes the migration across all organizations.
func (m *ResourceMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
ctx := context.Background()
orgs, err := m.getAllOrgs(sess)
if err != nil {
m.log.Error("failed to get organizations", "error", err)
return fmt.Errorf("failed to get organizations: %w", err)
}
if len(orgs) == 0 {
m.log.Info("No organizations found to migrate, skipping migration")
return nil
}
m.log.Info("Starting migration for all organizations", "org_count", len(orgs), "resources", m.resources)
for _, org := range orgs {
if err := m.migrateOrg(ctx, sess, org); err != nil {
return err
}
}
m.log.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
return nil
}
// migrateOrg handles migration for a single organization
func (m *ResourceMigration) migrateOrg(ctx context.Context, sess *xorm.Session, org orgInfo) error {
namespace := types.OrgNamespaceFormatter(org.ID)
m.log.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
// Create a service identity context for this namespace to authenticate with unified storage
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
startTime := time.Now()
opts := legacy.MigrateOptions{
Namespace: namespace,
Resources: m.resources,
WithHistory: true, // Migrate with full history
Progress: func(count int, msg string) {
m.log.Info("Migration progress", "org_id", org.ID, "count", count, "message", msg)
},
}
// Execute the migration via legacy migrator
response, err := m.migrator.Migrate(migrationCtx, opts)
if err != nil {
m.log.Error("Migration failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
}
// Validate the migration results
if err := m.validateMigration(sess, response); err != nil {
m.log.Error("Migration validation failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
return fmt.Errorf("migration validation failed for org %d (%s): %w", org.ID, org.Name, err)
}
m.log.Info("Migration completed for organization",
"org_id", org.ID,
"duration", time.Since(startTime),
"processed", response.Processed,
"summaries", len(response.Summary),
"rejected", len(response.Rejected))
return nil
}
// validateMigration calls the custom validation function if provided
func (m *ResourceMigration) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
if m.validationFunc == nil {
m.log.Debug("No validation function provided, skipping validation")
return nil
}
return m.validationFunc(sess, response, m.log)
}
// LegacyTableInfo defines how to map a unified storage resource to its legacy table
type LegacyTableInfo struct {
Table string // Legacy table name (e.g., "dashboard", "playlist")
WhereClause string // WHERE clause template with org_id parameter (e.g., "org_id = ? and is_folder = false")
}
// NewLegacyTableCountValidator creates a ValidationFunc that validates migration by comparing
// counts between legacy tables and unified storage.
//
// This is a helper for the common case of validating that all items from legacy tables
// were successfully migrated to unified storage.
//
// Parameters:
// - legacyTableMap: maps "group/resource" keys to LegacyTableInfo for validation.
// Only resources with mappings will be validated.
//
// Example:
//
// validator := NewLegacyTableCountValidator(map[string]LegacyTableInfo{
// "dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
// "folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
// })
func NewLegacyTableCountValidator(legacyTableMap map[string]LegacyTableInfo) ValidationFunc {
return func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error {
// Check for rejected items
if len(response.Rejected) > 0 {
log.Warn("Migration had rejected items", "count", len(response.Rejected))
for i, rejected := range response.Rejected {
if i < 10 { // Log first 10 rejected items
log.Warn("Rejected item",
"namespace", rejected.Key.Namespace,
"group", rejected.Key.Group,
"resource", rejected.Key.Resource,
"name", rejected.Key.Name,
"reason", rejected.Error)
}
}
// Rejections are not fatal - they may be expected for invalid data
}
// Validate counts for each resource type
for _, summary := range response.Summary {
key := fmt.Sprintf("%s/%s", summary.Group, summary.Resource)
tableInfo, ok := legacyTableMap[key]
if !ok {
log.Debug("No legacy table mapping for resource, skipping count validation",
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
"namespace", summary.Namespace)
continue
}
// Get legacy count
orgID, err := ParseOrgIDFromNamespace(summary.Namespace)
if err != nil {
return fmt.Errorf("invalid namespace %s: %w", summary.Namespace, err)
}
legacyCount, err := sess.Table(tableInfo.Table).Where(tableInfo.WhereClause, orgID).Count()
if err != nil {
return fmt.Errorf("failed to count %s: %w", tableInfo.Table, err)
}
// Account for rejected items in validation
expectedCount := summary.Count + int64(len(response.Rejected))
log.Info("Count validation",
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
"namespace", summary.Namespace,
"legacy_count", legacyCount,
"unified_count", summary.Count,
"rejected", len(response.Rejected),
"history", summary.History)
// Validate that we migrated all items (allowing for rejected items)
if legacyCount > expectedCount {
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
summary.Resource, summary.Group, summary.Namespace,
legacyCount, summary.Count, len(response.Rejected))
}
}
return nil
}
}
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
// Use authlib to properly parse all namespace formats including "default" for org 1
info, err := types.ParseNamespace(namespace)
if err != nil {
return 0, fmt.Errorf("failed to parse namespace: %w", err)
}
return info.OrgID, nil
}
// orgInfo represents basic organization information
type orgInfo struct {
ID int64 `xorm:"id"`
Name string `xorm:"name"`
}
// getAllOrgs retrieves all organizations from the database
func (m *ResourceMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
var orgs []orgInfo
err := sess.Table("org").Cols("id", "name").Find(&orgs)
if err != nil {
return nil, err
}
return orgs, nil
}
+118
View File
@@ -0,0 +1,118 @@
package migrations
import (
"context"
"fmt"
"os"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel"
"k8s.io/apimachinery/pkg/runtime/schema"
)
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
var logger = log.New("storage.unified.migrations")
type UnifiedStorageMigrationServiceImpl struct {
migrator UnifiedMigrator
cfg *setting.Cfg
sqlStore db.DB
kv kvstore.KVStore
}
var _ contract.UnifiedStorageMigrationService = (*UnifiedStorageMigrationServiceImpl)(nil)
// ProvideUnifiedStorageMigrationService is a Wire provider that creates the migration service.
// The service implements registry.BackgroundService and runs migrations during server startup.
func ProvideUnifiedStorageMigrationService(
migrator UnifiedMigrator,
cfg *setting.Cfg,
sqlStore db.DB,
kv kvstore.KVStore,
) contract.UnifiedStorageMigrationService {
return &UnifiedStorageMigrationServiceImpl{
migrator: migrator,
cfg: cfg,
sqlStore: sqlStore,
kv: kv,
}
}
// Run executes unified storage migrations as a background service.
// This blocks until migrations complete. If migrations fail, an error is returned
// which will prevent Grafana from starting.
func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error {
// TODO: temporary skip migrations in test environments to prevent integration test timeouts.
if os.Getenv("GRAFANA_TEST_DB") != "" {
return nil
}
// skip migrations if disabled in config
if p.cfg.DisableDataMigrations {
logger.Info("Data migrations are disabled, skipping")
return nil
}
// TODO: Re-enable once migrations are ready
// TODO: add guarantee that this only runs once
// return RegisterMigrations(p.migrator, p.cfg, p.sqlStore)
return nil
}
// RegisterMigrations initializes and registers all unified storage migrations.
// This function is the entry point for all data migrations from legacy storage
// to unified storage. It returns an error if migrations fail, preventing Grafana
// from starting with inconsistent data.
func RegisterMigrations(
migrator UnifiedMigrator,
cfg *setting.Cfg,
sqlStore db.DB,
) error {
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
defer span.End()
mg := sqlstoremigrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unifiedstorage")
mg.AddCreateMigration()
if err := prometheus.Register(mg); err != nil {
logger.Warn("Failed to register migrator metrics", "error", err)
}
// Register resource migrations
// To add a new resource type, simply add another migration here with the appropriate resources
registerResourceMigrations(mg, migrator)
// Run all registered migrations (blocking)
sec := cfg.Raw.Section("database")
if err := mg.RunMigrations(ctx,
sec.Key("migration_locking").MustBool(true),
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
return fmt.Errorf("unified storage data migration failed: %w", err)
}
logger.Info("Unified storage migrations completed successfully")
return nil
}
// registerResourceMigrations registers all unified storage resource migrations.
// Add new resource types here by creating additional ResourceMigration instances.
func registerResourceMigrations(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator) {
dashboardsAndFolders := NewResourceMigration(
migrator,
[]schema.GroupResource{
{Group: "folder.grafana.app", Resource: "folders"},
{Group: "dashboard.grafana.app", Resource: "dashboards"},
},
"folders-dashboards",
NewLegacyTableCountValidator(map[string]LegacyTableInfo{
"folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
"dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
}),
)
mg.AddMigration("folders and dashboards migration", dashboardsAndFolders)
}