feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
co-authored by
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -1,70 +0,0 @@
|
||||
package datamigrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
|
||||
)
|
||||
|
||||
var (
|
||||
_ provisioning.ProvisioningService = (*stubProvisioning)(nil)
|
||||
)
|
||||
|
||||
func newStubProvisioning(path string) (provisioning.ProvisioningService, error) {
|
||||
cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stub := &stubProvisioning{
|
||||
path: make(map[string]string),
|
||||
}
|
||||
for _, cfg := range cfgs {
|
||||
stub.path[cfg.Name] = cfg.Options["path"].(string)
|
||||
}
|
||||
return &stubProvisioning{}, nil
|
||||
}
|
||||
|
||||
type stubProvisioning struct {
|
||||
path map[string]string // name > options.path
|
||||
}
|
||||
|
||||
// GetAllowUIUpdatesFromConfig implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
|
||||
return s.path[name]
|
||||
}
|
||||
|
||||
// ProvisionAlerting implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionAlerting(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionDashboards implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionDashboards(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionDatasources implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionDatasources(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionPlugins implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionPlugins(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// Run implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) Run(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// RunInitProvisioners implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) RunInitProvisioners(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
@@ -24,10 +24,11 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol/acimpl"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/parquet"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
@@ -52,7 +53,6 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
{Group: folders.GROUP, Resource: folders.RESOURCE},
|
||||
{Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE},
|
||||
},
|
||||
LargeObjects: nil, // TODO... from config
|
||||
Progress: func(count int, msg string) {
|
||||
const minInterval = time.Second
|
||||
shouldPrint := count < 1 || time.Since(last) > minInterval
|
||||
@@ -69,31 +69,26 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
}
|
||||
featureToggles := featuremgmt.ProvideToggles(featureManager)
|
||||
|
||||
provisioning, err := newStubProvisioning(cfg.ProvisioningPath)
|
||||
provisioning, err := provisioning.ProvideStubProvisioningService(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrator := legacy.NewDashboardAccess(
|
||||
grpcClient, err := newUnifiedClient(cfg, sqlStore, featureToggles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dashboardAccess := legacy.ProvideMigratorDashboardAccessor(
|
||||
legacysql.NewDatabaseProvider(sqlStore),
|
||||
authlib.OrgNamespaceFormatter,
|
||||
nil, // no dashboards.Store
|
||||
provisioning,
|
||||
nil, // no librarypanels.Service
|
||||
sort.ProvideService(),
|
||||
nil, // we don't delete during migration, and this is only need to delete permission.
|
||||
acimpl.ProvideAccessControl(featuremgmt.WithFeatures()),
|
||||
featureToggles,
|
||||
)
|
||||
|
||||
client, err := newUnifiedClient(cfg, sqlStore, featureToggles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.Bool("non-interactive") {
|
||||
opts.Store = client
|
||||
opts.BlobStore = client
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
|
||||
|
||||
opts.WithHistory = true // always include history in non-interactive mode
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if exitErr := handleMigrationError(err, rsp); exitErr != nil {
|
||||
@@ -113,6 +108,8 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return err
|
||||
}
|
||||
if yes {
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, nil) // no need for grpc client for counting
|
||||
|
||||
opts.OnlyCount = true
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
@@ -141,12 +138,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
opts.Store, err = newParquetClient(file)
|
||||
parquetClient, err := newParquetClient(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
migrator := migrations.ProvideUnifiedMigratorParquet(dashboardAccess, parquetClient)
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -172,7 +170,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
req.Kinds = append(req.Kinds, fmt.Sprintf("%s/%s", r.Group, r.Resource))
|
||||
}
|
||||
|
||||
stats, err := client.GetStats(ctx, req)
|
||||
stats, err := grpcClient.GetStats(ctx, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,10 +186,9 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return err
|
||||
}
|
||||
if yes {
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
opts.Store = client
|
||||
opts.BlobStore = client
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockLegacyMigrator is an autogenerated mock type for the LegacyMigrator type
|
||||
type MockLegacyMigrator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockLegacyMigrator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockLegacyMigrator) EXPECT() *MockLegacyMigrator_Expecter {
|
||||
return &MockLegacyMigrator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Migrate provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockLegacyMigrator) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Migrate")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockLegacyMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
|
||||
type MockLegacyMigrator_Migrate_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Migrate is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockLegacyMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockLegacyMigrator_Migrate_Call {
|
||||
return &MockLegacyMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockLegacyMigrator creates a new instance of MockLegacyMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockLegacyMigrator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockLegacyMigrator {
|
||||
mock := &MockLegacyMigrator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -1,460 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"google.golang.org/grpc/metadata"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Store resourcepb.BulkStoreClient
|
||||
LargeObjects apistore.LargeObjectSupport
|
||||
BlobStore resourcepb.BlobStoreClient
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
// Read from legacy and write into unified storage
|
||||
//
|
||||
//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter
|
||||
type LegacyMigrator interface {
|
||||
Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
}
|
||||
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideLegacyMigrator(
|
||||
sql db.DB, // direct access to tables
|
||||
provisioning provisioning.ProvisioningService, // only needed for dashboard settings
|
||||
libraryPanelSvc librarypanels.Service,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) LegacyMigrator {
|
||||
dbp := legacysql.NewDatabaseProvider(sql)
|
||||
return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), dashboardPermissionSvc, accessControl, features)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
// migrate function -- works for a single kind
|
||||
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
|
||||
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
info, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts.Progress == nil {
|
||||
opts.Progress = func(count int, msg string) {} // noop
|
||||
}
|
||||
|
||||
// Migrate everything
|
||||
if len(opts.Resources) < 1 {
|
||||
return nil, fmt.Errorf("missing resource selector")
|
||||
}
|
||||
|
||||
migratorFuncs := []migratorFunc{}
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateFolders)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
migratorFuncs = append(migratorFuncs, a.migratePanels)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
})
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported resource: %s", res)
|
||||
}
|
||||
}
|
||||
if opts.OnlyCount {
|
||||
return a.countValues(ctx, opts)
|
||||
}
|
||||
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
if md, ok := metadata.FromOutgoingContext(ctx); ok {
|
||||
a.log.Debug("bulk grpc request metadata",
|
||||
"metadata", md,
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
} else {
|
||||
a.log.Debug("bulk grpc request, no metadata found",
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
}
|
||||
|
||||
stream, err := opts.Store.BulkProcess(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now run each migration
|
||||
blobStore := BlobStoreInfo{}
|
||||
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
||||
for _, m := range migratorFuncs {
|
||||
blobs, err := m(ctx, info.OrgID, opts, stream)
|
||||
if err != nil {
|
||||
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
||||
return nil, err
|
||||
}
|
||||
if blobs != nil {
|
||||
blobStore.Count += blobs.Count
|
||||
blobStore.Size += blobs.Size
|
||||
}
|
||||
}
|
||||
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
||||
return stream.CloseAndRecv()
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
large := opts.LargeObjects
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
// With large object support
|
||||
if large != nil && len(body) > large.Threshold() {
|
||||
obj, err := utils.MetaAccessor(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body)))
|
||||
err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// The smaller version (most of spec removed)
|
||||
req.Value, err = json.Marshal(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
blobs.Count++
|
||||
blobs.Size += int64(len(body))
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockMigrationDashboardAccessor is an autogenerated mock type for the MigrationDashboardAccessor type
|
||||
type MockMigrationDashboardAccessor struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockMigrationDashboardAccessor_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockMigrationDashboardAccessor) EXPECT() *MockMigrationDashboardAccessor_Expecter {
|
||||
return &MockMigrationDashboardAccessor_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// CountResources provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockMigrationDashboardAccessor) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CountResources")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_CountResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountResources'
|
||||
type MockMigrationDashboardAccessor_CountResources_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CountResources is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) CountResources(ctx interface{}, opts interface{}) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
return &MockMigrationDashboardAccessor_CountResources_Call{Call: _e.mock.On("CountResources", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateDashboards provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateDashboards")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateDashboards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateDashboards'
|
||||
type MockMigrationDashboardAccessor_MigrateDashboards_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateDashboards is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateDashboards(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateDashboards_Call{Call: _e.mock.On("MigrateDashboards", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateFolders provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateFolders")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateFolders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateFolders'
|
||||
type MockMigrationDashboardAccessor_MigrateFolders_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateFolders is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateFolders(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateFolders_Call{Call: _e.mock.On("MigrateFolders", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateLibraryPanels")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateLibraryPanels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateLibraryPanels'
|
||||
type MockMigrationDashboardAccessor_MigrateLibraryPanels_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateLibraryPanels(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateLibraryPanels_Call{Call: _e.mock.On("MigrateLibraryPanels", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockMigrationDashboardAccessor creates a new instance of MockMigrationDashboardAccessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockMigrationDashboardAccessor(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockMigrationDashboardAccessor {
|
||||
mock := &MockMigrationDashboardAccessor{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"go.opentelemetry.io/otel"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/utils/ptr"
|
||||
|
||||
claims "github.com/grafana/authlib/types"
|
||||
@@ -20,6 +23,7 @@ import (
|
||||
dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
|
||||
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
@@ -35,16 +39,30 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
var (
|
||||
_ DashboardAccess = (*dashboardSqlAccess)(nil)
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
type dashboardRow struct {
|
||||
// The numeric version for this dashboard
|
||||
RV int64
|
||||
@@ -63,8 +81,9 @@ type dashboardRow struct {
|
||||
type dashboardSqlAccess struct {
|
||||
sql legacysql.LegacyDatabaseProvider
|
||||
namespacer request.NamespaceMapper
|
||||
provisioning provisioning.ProvisioningService
|
||||
provisioning provisioning.StubProvisioningService
|
||||
|
||||
// TODO: consider enabling this by default for on-prem migrations
|
||||
invalidDashboardParseFallbackEnabled bool
|
||||
|
||||
// Use for writing (not reading)
|
||||
@@ -73,7 +92,7 @@ type dashboardSqlAccess struct {
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService
|
||||
|
||||
accessControl accesscontrol.AccessControl
|
||||
libraryPanelSvc librarypanels.Service
|
||||
libraryPanelSvc librarypanels.Service // only used for save dashboard
|
||||
|
||||
// Typically one... the server wrapper
|
||||
subscribers []chan *resource.WrittenEvent
|
||||
@@ -81,7 +100,27 @@ type dashboardSqlAccess struct {
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
// ProvideMigratorDashboardAccessor creates a DashboardAccess specifically for migration purposes.
|
||||
// This provider is used by Wire DI and only includes the minimal dependencies needed for migrations.
|
||||
func ProvideMigratorDashboardAccessor(
|
||||
sql legacysql.LegacyDatabaseProvider,
|
||||
provisioning provisioning.StubProvisioningService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) MigrationDashboardAccessor {
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
namespacer: claims.OrgNamespaceFormatter,
|
||||
dashStore: nil, // not needed for migration
|
||||
provisioning: provisioning,
|
||||
dashboardPermissionSvc: nil, // not needed for migration
|
||||
libraryPanelSvc: nil, // not needed for migration
|
||||
accessControl: accessControl,
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
|
||||
func NewDashboardSQLAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
namespacer request.NamespaceMapper,
|
||||
dashStore dashboards.Store,
|
||||
provisioning provisioning.ProvisioningService,
|
||||
@@ -90,7 +129,7 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) DashboardAccess {
|
||||
) *dashboardSqlAccess {
|
||||
dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter)
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
@@ -101,7 +140,6 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc: dashboardPermissionSvc,
|
||||
libraryPanelSvc: libraryPanelSvc,
|
||||
accessControl: accessControl,
|
||||
log: log.New("dashboard.legacysql"),
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
@@ -149,6 +187,290 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
|
||||
}, err
|
||||
}
|
||||
|
||||
// CountResources counts resources without migrating them
|
||||
func (a *dashboardSqlAccess) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := claims.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
// MigrateDashboards handles the dashboard migration logic
|
||||
func (a *dashboardSqlAccess) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboardV1.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// MigrateFolders handles the folder migration logic
|
||||
func (a *dashboardSqlAccess) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels handles the library panel migration logic
|
||||
func (a *dashboardSqlAccess) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var _ resource.ListIterator = (*rowsWrapper)(nil)
|
||||
|
||||
type rowsWrapper struct {
|
||||
|
||||
@@ -21,6 +21,9 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/dashboards"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestScanRow(t *testing.T) {
|
||||
@@ -529,3 +532,146 @@ func TestParseLibraryPanelRow(t *testing.T) {
|
||||
require.Nil(t, updatedTimestamp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -55,10 +55,9 @@ type LibraryPanelQuery struct {
|
||||
LastID int64
|
||||
}
|
||||
|
||||
type DashboardAccess interface {
|
||||
type DashboardAccessor interface {
|
||||
resource.StorageBackend
|
||||
resourcepb.ResourceIndexServer
|
||||
LegacyMigrator
|
||||
|
||||
GetDashboard(ctx context.Context, orgId int64, uid string, version int64) (*dashboardV1.Dashboard, int64, error)
|
||||
SaveDashboard(ctx context.Context, orgId int64, dash *dashboardV1.Dashboard, failOnExisting bool) (*dashboardV1.Dashboard, bool, error)
|
||||
@@ -67,3 +66,12 @@ type DashboardAccess interface {
|
||||
// Get a typed list
|
||||
GetLibraryPanels(ctx context.Context, query LibraryPanelQuery) (*dashboardV0.LibraryPanelList, error)
|
||||
}
|
||||
|
||||
//go:generate mockery --name MigrationDashboardAccessor --structname MockMigrationDashboardAccessor --inpackage --filename migration_dashboard_accessor_mock.go --with-expecter
|
||||
type MigrationDashboardAccessor interface {
|
||||
// Migration helper methods - these support the separate LegacyMigrator
|
||||
CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
type DashboardStorage struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
DashboardService dashboards.DashboardService
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ var (
|
||||
)
|
||||
|
||||
type LibraryPanelStore struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
ResourceInfo utils.ResourceInfo
|
||||
service libraryelements.Service
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ func RegisterAPIService(
|
||||
publicDashboardService: publicDashboardService,
|
||||
|
||||
legacy: &DashboardStorage{
|
||||
Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
Access: legacy.NewDashboardSQLAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
DashboardService: dashboardService,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ type dtoBuilder = func(dashboard runtime.Object, access *dashboard.DashboardAcce
|
||||
// The DTO returns everything the UI needs in a single request
|
||||
type DTOConnector struct {
|
||||
getter rest.Getter
|
||||
legacy legacy.DashboardAccess
|
||||
legacy legacy.DashboardAccessor
|
||||
unified resource.ResourceClient
|
||||
largeObjects apistore.LargeObjectSupport
|
||||
accessControl accesscontrol.AccessControl
|
||||
@@ -42,7 +42,7 @@ type DTOConnector struct {
|
||||
func NewDTOConnector(
|
||||
getter rest.Getter,
|
||||
largeObjects apistore.LargeObjectSupport,
|
||||
legacyAccess legacy.DashboardAccess,
|
||||
legacyAccess legacy.DashboardAccessor,
|
||||
resourceClient resource.ResourceClient,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
scheme *runtime.Scheme,
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources/signature"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/parquet"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
@@ -29,7 +30,7 @@ type LegacyResourcesMigrator interface {
|
||||
type legacyResourcesMigrator struct {
|
||||
repositoryResources resources.RepositoryResourcesFactory
|
||||
parsers resources.ParserFactory
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
signerFactory signature.SignerFactory
|
||||
clients resources.ClientFactory
|
||||
exportFn export.ExportFn
|
||||
@@ -38,7 +39,7 @@ type legacyResourcesMigrator struct {
|
||||
func NewLegacyResourcesMigrator(
|
||||
repositoryResources resources.RepositoryResourcesFactory,
|
||||
parsers resources.ParserFactory,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
signerFactory signature.SignerFactory,
|
||||
clients resources.ClientFactory,
|
||||
exportFn export.ExportFn,
|
||||
@@ -46,7 +47,7 @@ func NewLegacyResourcesMigrator(
|
||||
return &legacyResourcesMigrator{
|
||||
repositoryResources: repositoryResources,
|
||||
parsers: parsers,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
signerFactory: signerFactory,
|
||||
clients: clients,
|
||||
exportFn: exportFn,
|
||||
@@ -94,7 +95,7 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
|
||||
reader := newLegacyResourceMigrator(
|
||||
rw,
|
||||
m.legacyMigrator,
|
||||
m.dashboardAccess,
|
||||
parser,
|
||||
repositoryResources,
|
||||
progress,
|
||||
@@ -113,21 +114,21 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
}
|
||||
|
||||
type legacyResourceResourceMigrator struct {
|
||||
repo repository.ReaderWriter
|
||||
legacy legacy.LegacyMigrator
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
repo repository.ReaderWriter
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
}
|
||||
|
||||
func newLegacyResourceMigrator(
|
||||
repo repository.ReaderWriter,
|
||||
legacy legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
parser resources.Parser,
|
||||
resources resources.RepositoryResources,
|
||||
progress jobs.JobProgressRecorder,
|
||||
@@ -141,16 +142,16 @@ func newLegacyResourceMigrator(
|
||||
history = make(map[string]string)
|
||||
}
|
||||
return &legacyResourceResourceMigrator{
|
||||
repo: repo,
|
||||
legacy: legacy,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
repo: repo,
|
||||
dashboardAccess: dashboardAccess,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,14 +226,21 @@ func (r *legacyResourceResourceMigrator) Write(ctx context.Context, key *resourc
|
||||
|
||||
func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
r.progress.SetMessage(ctx, fmt.Sprintf("migrate %s resource", r.kind.Resource))
|
||||
|
||||
// Create a parquet migrator with this instance as the BulkResourceWriter
|
||||
parquetClient := parquet.NewBulkResourceWriterClient(r)
|
||||
migrator := unifiedmigrations.ProvideUnifiedMigratorParquet(
|
||||
r.dashboardAccess,
|
||||
parquetClient,
|
||||
)
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: r.namespace,
|
||||
WithHistory: r.options.History,
|
||||
Resources: []schema.GroupResource{r.kind},
|
||||
Store: parquet.NewBulkResourceWriterClient(r),
|
||||
OnlyCount: true, // first get the count
|
||||
}
|
||||
stats, err := r.legacy.Migrate(ctx, opts)
|
||||
stats, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to count legacy items %w", err)
|
||||
}
|
||||
@@ -248,7 +256,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
opts.OnlyCount = false // this time actually write
|
||||
_, err = r.legacy.Migrate(ctx, opts)
|
||||
_, err = migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate legacy %s: %w", r.kind.Resource, err)
|
||||
}
|
||||
|
||||
@@ -93,8 +93,8 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
mockRepoResourcesFactory.On("Client", mock.Anything, mock.Anything).
|
||||
Return(mockRepoResources, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("legacy migrator error"))
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
signerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -136,7 +136,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
@@ -308,22 +308,18 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
History: true,
|
||||
}).Return(mockSigner, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
// Mock CountResources for the count phase
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once()
|
||||
// Mock MigrateDashboards for the actual migration phase (dashboards resource)
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Count: 10,
|
||||
History: 5,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Migration phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{
|
||||
Count: 10,
|
||||
Size: 5,
|
||||
}, nil).Once()
|
||||
|
||||
mockClients := resources.NewMockResourceClients(t)
|
||||
mockClientFactory := resources.NewMockClientFactory(t)
|
||||
@@ -339,7 +335,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
mockSignerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -362,7 +358,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
@@ -742,8 +738,8 @@ func TestLegacyResourceResourceMigrator_Write(t *testing.T) {
|
||||
|
||||
func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
t.Run("should fail when legacy migrate count fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("count error"))
|
||||
|
||||
@@ -752,7 +748,7 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
@@ -766,89 +762,91 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to count legacy items")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should fail when legacy migrate write fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
// For test-resources GroupResource, we don't know which method it will call, but since it's not dashboards/folders/librarypanels,
|
||||
// the Migrate will fail trying to map the resource type. Let's make it dashboards for this test.
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("write error")).Once() // Write phase
|
||||
}), mock.Anything).Return(nil, errors.New("write error")).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "test-resources"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "migrate legacy test-resources: write error")
|
||||
require.Contains(t, err.Error(), "migrate legacy dashboards: write error")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should successfully migrate resource", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to history if history is greater than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 1,
|
||||
History: 100,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -856,39 +854,39 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to count if history is less than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 200,
|
||||
History: 1,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -897,20 +895,20 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signer,
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
@@ -109,7 +110,7 @@ type APIBuilder struct {
|
||||
jobHistoryConfig *JobHistoryConfig
|
||||
jobHistoryLoki *jobs.LokiJobHistory
|
||||
resourceLister resources.ResourceLister
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
storageStatus dualwrite.Service
|
||||
unified resource.ResourceClient
|
||||
repoFactory repository.Factory
|
||||
@@ -135,7 +136,7 @@ func NewAPIBuilder(
|
||||
features featuremgmt.FeatureToggles,
|
||||
unified resource.ResourceClient,
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
access authlib.AccessChecker,
|
||||
@@ -158,6 +159,7 @@ func NewAPIBuilder(
|
||||
clients = resources.NewClientFactory(configProvider)
|
||||
}
|
||||
parsers := resources.NewParserFactory(clients)
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(dashboardAccess, unified)
|
||||
resourceLister := resources.NewResourceListerForMigrations(unified, legacyMigrator, storageStatus)
|
||||
|
||||
b := &APIBuilder{
|
||||
@@ -170,7 +172,7 @@ func NewAPIBuilder(
|
||||
parsers: parsers,
|
||||
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
|
||||
resourceLister: resourceLister,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
storageStatus: storageStatus,
|
||||
unified: unified,
|
||||
access: access,
|
||||
@@ -234,7 +236,7 @@ func RegisterAPIService(
|
||||
client resource.ResourceClient, // implements resource.RepositoryClient
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
access authlib.AccessClient,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
tracer tracing.Tracer,
|
||||
@@ -258,7 +260,7 @@ func RegisterAPIService(
|
||||
features,
|
||||
client,
|
||||
configProvider,
|
||||
legacyMigrator, storageStatus,
|
||||
dashboardAccess, storageStatus,
|
||||
usageStats,
|
||||
access,
|
||||
tracer,
|
||||
@@ -722,7 +724,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
|
||||
legacyResources := migrate.NewLegacyResourcesMigrator(
|
||||
b.repositoryResources,
|
||||
b.parsers,
|
||||
b.legacyMigrator,
|
||||
b.dashboardAccess,
|
||||
signerFactory,
|
||||
b.clients,
|
||||
export.ExportAll,
|
||||
@@ -1241,8 +1243,9 @@ func (b *APIBuilder) tryRunningOnlyUnifiedStorage() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Count how many things exist
|
||||
rsp, err := b.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
// Count how many things exist - create a migrator on-demand for this
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(b.dashboardAccess, b.unified)
|
||||
rsp, err := legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: "default", // FIXME! this works for single org, but need to check multi-org
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
@@ -30,9 +31,9 @@ type ResourceStore interface {
|
||||
}
|
||||
|
||||
type ResourceListerFromSearch struct {
|
||||
store ResourceStore
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
storageStatus dualwrite.Service
|
||||
store ResourceStore
|
||||
migrator migrations.UnifiedMigrator
|
||||
storageStatus dualwrite.Service
|
||||
}
|
||||
|
||||
func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
@@ -42,13 +43,13 @@ func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
// FIXME: the logic about migration and storage should probably be separated from this
|
||||
func NewResourceListerForMigrations(
|
||||
store ResourceStore,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
migrator migrations.UnifiedMigrator,
|
||||
storageStatus dualwrite.Service,
|
||||
) ResourceLister {
|
||||
return &ResourceListerFromSearch{
|
||||
store: store,
|
||||
legacyMigrator: legacyMigrator,
|
||||
storageStatus: storageStatus,
|
||||
store: store,
|
||||
migrator: migrator,
|
||||
storageStatus: storageStatus,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,8 +134,8 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit
|
||||
}
|
||||
|
||||
// Get the stats based on what a migration could support
|
||||
if o.storageStatus != nil && o.legacyMigrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
if o.storageStatus != nil && o.migrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.migrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
@@ -48,7 +48,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl"
|
||||
"github.com/grafana/grafana/pkg/services/team/teamapi"
|
||||
"github.com/grafana/grafana/pkg/services/updatemanager"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
)
|
||||
|
||||
func ProvideBackgroundServiceRegistry(
|
||||
@@ -74,7 +73,6 @@ func ProvideBackgroundServiceRegistry(
|
||||
dashboardServiceImpl *service.DashboardServiceImpl,
|
||||
secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker,
|
||||
fixedRolesLoader *accesscontrol.FixedRolesLoader,
|
||||
unifiedStorageMigrationProvider unifiedmigrations.UnifiedStorageMigrationProvider,
|
||||
// Need to make sure these are initialized, is there a better place to put them?
|
||||
_ dashboardsnapshots.Service,
|
||||
_ serviceaccounts.Service,
|
||||
@@ -91,7 +89,6 @@ func ProvideBackgroundServiceRegistry(
|
||||
notifications,
|
||||
rendering,
|
||||
tokenService,
|
||||
unifiedStorageMigrationProvider,
|
||||
provisioning,
|
||||
grafanaUpdateChecker,
|
||||
pluginsUpdateChecker,
|
||||
|
||||
+5
-3
@@ -128,6 +128,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginaccesscontrol"
|
||||
"github.com/grafana/grafana/pkg/services/preference/prefimpl"
|
||||
promTypeMigration "github.com/grafana/grafana/pkg/services/promtypemigration"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/publicdashboards"
|
||||
publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api"
|
||||
publicdashboardsStore "github.com/grafana/grafana/pkg/services/publicdashboards/database"
|
||||
@@ -238,7 +239,9 @@ var wireBasicSet = wire.NewSet(
|
||||
uss.ProvideService,
|
||||
wire.Bind(new(usagestats.Service), new(*uss.UsageStats)),
|
||||
validator.ProvideService,
|
||||
legacy.ProvideLegacyMigrator,
|
||||
provisioning.ProvideStubProvisioningService,
|
||||
legacy.ProvideMigratorDashboardAccessor,
|
||||
unifiedmigrations.ProvideUnifiedMigrator,
|
||||
pluginsintegration.WireSet,
|
||||
pluginDashboards.ProvideFileStoreManager,
|
||||
wire.Bind(new(pluginDashboards.FileStore), new(*pluginDashboards.FileStoreManager)),
|
||||
@@ -466,8 +469,7 @@ var wireBasicSet = wire.NewSet(
|
||||
// Unified storage
|
||||
resource.ProvideStorageMetrics,
|
||||
resource.ProvideIndexMetrics,
|
||||
unifiedmigrations.ProvideUnifiedStorageMigrationProvider,
|
||||
wire.Bind(new(unifiedmigrations.UnifiedStorageMigrationProvider), new(*unifiedmigrations.UnifiedStorageMigrationProviderImpl)),
|
||||
unifiedmigrations.ProvideUnifiedStorageMigrationService,
|
||||
// Kubernetes API server
|
||||
grafanaapiserver.WireSet,
|
||||
apiregistry.WireSet,
|
||||
|
||||
Generated
+30
-17
File diff suppressed because one or more lines are too long
@@ -62,6 +62,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/services/validations"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
search2 "github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
@@ -98,6 +99,7 @@ var wireExtsBasicSet = wire.NewSet(
|
||||
wire.Bind(new(validations.DataSourceRequestURLValidator), new(*validations.OSSDataSourceRequestURLValidator)),
|
||||
provisioning.ProvideService,
|
||||
wire.Bind(new(provisioning.ProvisioningService), new(*provisioning.ProvisioningServiceImpl)),
|
||||
legacysql.NewDatabaseProvider,
|
||||
backgroundsvcs.ProvideBackgroundServiceRegistry,
|
||||
wire.Bind(new(registry.BackgroundServiceRegistry), new(*backgroundsvcs.BackgroundServiceRegistry)),
|
||||
migrations.ProvideOSSMigrations,
|
||||
|
||||
@@ -107,7 +107,7 @@ func NewDualWriter(
|
||||
if currentMode != mode {
|
||||
klog.Warningf("Requested DualWrite mode: %d, but using %d for %+v", mode, currentMode, gr)
|
||||
}
|
||||
return dualwrite.NewDualWriter(gr, currentMode, legacy, storage)
|
||||
return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage)
|
||||
}
|
||||
|
||||
func getRequestInfo(gr schema.GroupResource, namespaceMapper request.NamespaceMapper) *k8srequest.RequestInfo {
|
||||
|
||||
@@ -379,7 +379,7 @@ func InstallAPIs(
|
||||
case grafanarest.Mode4, grafanarest.Mode5:
|
||||
return storage, nil
|
||||
default:
|
||||
return dualwrite.NewDualWriter(gr, currentMode, legacy, storage)
|
||||
return dualwrite.NewStaticStorage(gr, currentMode, legacy, storage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ type service struct {
|
||||
|
||||
authorizer *authorizer.GrafanaAuthorizer
|
||||
serverLockService builder.ServerLockService
|
||||
storageStatus dualwrite.Service
|
||||
dualWriter dualwrite.Service
|
||||
kvStore kvstore.KVStore
|
||||
|
||||
pluginClient plugins.Client
|
||||
@@ -127,7 +127,7 @@ func ProvideService(
|
||||
datasources datasource.ScopedPluginDatasourceProvider,
|
||||
contextProvider datasource.PluginContextWrapper,
|
||||
pluginStore pluginstore.Store,
|
||||
storageStatus dualwrite.Service,
|
||||
dualWriter dualwrite.Service,
|
||||
unified resource.ResourceClient,
|
||||
secrets secret.InlineSecureValueSupport,
|
||||
restConfigProvider RestConfigProvider,
|
||||
@@ -158,7 +158,7 @@ func ProvideService(
|
||||
contextProvider: contextProvider,
|
||||
pluginStore: pluginStore,
|
||||
serverLockService: serverLockService,
|
||||
storageStatus: storageStatus,
|
||||
dualWriter: dualWriter,
|
||||
unified: unified,
|
||||
secrets: secrets,
|
||||
restConfigProvider: restConfigProvider,
|
||||
@@ -385,12 +385,17 @@ func (s *service) start(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Install the API group+version for existing builders
|
||||
err = builder.InstallAPIs(s.scheme, s.codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions,
|
||||
err = builder.InstallAPIs(s.scheme,
|
||||
s.codecs,
|
||||
server,
|
||||
serverConfig.RESTOptionsGetter,
|
||||
builders,
|
||||
o.StorageOptions,
|
||||
s.metrics,
|
||||
request.GetNamespaceMapper(s.cfg),
|
||||
kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
|
||||
s.serverLockService,
|
||||
s.storageStatus,
|
||||
s.dualWriter,
|
||||
optsregister,
|
||||
s.features,
|
||||
s.dualWriterMetrics,
|
||||
@@ -409,7 +414,7 @@ func (s *service) start(ctx context.Context) error {
|
||||
kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
|
||||
s.serverLockService,
|
||||
request.GetNamespaceMapper(s.cfg),
|
||||
s.storageStatus,
|
||||
s.dualWriter,
|
||||
s.dualWriterMetrics,
|
||||
s.builderMetrics,
|
||||
serverConfig.MergedResourceConfig,
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package provisioning
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
type StubProvisioningService interface {
|
||||
GetDashboardProvisionerResolvedPath(name string) string
|
||||
GetAllowUIUpdatesFromConfig(name string) bool
|
||||
}
|
||||
|
||||
func ProvideStubProvisioningService(cfg *setting.Cfg) (StubProvisioningService, error) {
|
||||
return NewStubProvisioning(cfg.ProvisioningPath)
|
||||
}
|
||||
|
||||
func NewStubProvisioning(path string) (StubProvisioningService, error) {
|
||||
cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stub := &stubProvisioning{
|
||||
path: make(map[string]string),
|
||||
allowUIUpdates: make(map[string]bool),
|
||||
log: log.New("provisioning.stub"),
|
||||
}
|
||||
for _, cfg := range cfgs {
|
||||
stub.path[cfg.Name] = cfg.Options["path"].(string)
|
||||
stub.allowUIUpdates[cfg.Name] = cfg.AllowUIUpdates
|
||||
}
|
||||
return stub, nil
|
||||
}
|
||||
|
||||
type stubProvisioning struct {
|
||||
path map[string]string // name > options.path
|
||||
allowUIUpdates map[string]bool
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
|
||||
return s.allowUIUpdates[name]
|
||||
}
|
||||
|
||||
func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
|
||||
path := s.path[name]
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
s.log.Warn("Cannot read directory", "error", err)
|
||||
}
|
||||
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
s.log.Warn("Could not create absolute path", "path", path, "error", err)
|
||||
}
|
||||
|
||||
path, err = filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
s.log.Warn("Failed to read content of symlinked path", "path", path, "error", err)
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
path = s.path[name]
|
||||
s.log.Info("falling back to original path due to EvalSymlink/Abs failure")
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -71,7 +71,7 @@ func TestMode1_Create(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
|
||||
@@ -154,7 +154,7 @@ func TestMode1_Get(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{})
|
||||
@@ -217,7 +217,7 @@ func TestMode1_List(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = dw.List(context.Background(), &metainternalversion.ListOptions{})
|
||||
@@ -286,7 +286,7 @@ func TestMode1_Delete(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Delete(context.Background(), name, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
|
||||
@@ -361,7 +361,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, tt.input)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
|
||||
@@ -434,7 +434,7 @@ func TestMode1_Update(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode1, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode1, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestMode2_Create(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, tt.input)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
|
||||
@@ -154,7 +154,7 @@ func TestMode2_Get(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, tt.input)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
|
||||
@@ -229,7 +229,7 @@ func TestMode2_List(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
|
||||
@@ -331,7 +331,7 @@ func TestMode2_Delete(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
|
||||
@@ -401,7 +401,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{})
|
||||
@@ -471,7 +471,7 @@ func TestMode2_Update(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode2, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode2, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestMode3_Create(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, tt.input)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
|
||||
@@ -134,7 +134,7 @@ func TestMode3_Get(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.Get(context.Background(), name, &metav1.GetOptions{})
|
||||
@@ -185,7 +185,7 @@ func TestMode3_List(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}})
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
res, err := dw.List(context.Background(), &metainternalversion.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "foo"}})
|
||||
@@ -276,7 +276,7 @@ func TestMode3_Delete(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Delete(context.Background(), name, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
|
||||
@@ -346,7 +346,7 @@ func TestMode3_DeleteCollection(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: name}}, &metainternalversion.ListOptions{})
|
||||
@@ -416,7 +416,7 @@ func TestMode3_Update(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
dw, err := NewDualWriter(kind, rest.Mode3, ls, us)
|
||||
dw, err := NewStaticStorage(kind, rest.Mode3, ls, us)
|
||||
require.NoError(t, err)
|
||||
|
||||
obj, _, err := dw.Update(context.Background(), name, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
|
||||
|
||||
@@ -14,37 +14,6 @@ import (
|
||||
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
|
||||
)
|
||||
|
||||
func (m *service) NewStorage(gr schema.GroupResource, legacy grafanarest.Storage, unified grafanarest.Storage) (grafanarest.Storage, error) {
|
||||
status, err := m.Status(context.Background(), gr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if m.enabled && status.Runtime {
|
||||
// Dynamic storage behavior
|
||||
return &runtimeDualWriter{
|
||||
service: m,
|
||||
legacy: legacy,
|
||||
unified: unified,
|
||||
dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read
|
||||
gr: gr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if status.ReadUnified {
|
||||
if status.WriteLegacy {
|
||||
// Write both, read unified
|
||||
return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil
|
||||
}
|
||||
return unified, nil
|
||||
}
|
||||
if status.WriteUnified {
|
||||
// Write both, read legacy
|
||||
return &dualWriter{legacy: legacy, unified: unified}, nil
|
||||
}
|
||||
return legacy, nil
|
||||
}
|
||||
|
||||
// The runtime dual writer implements the various modes we have described as: mode:1/2/3/4/5
|
||||
// However the behavior can be configured at runtime rather than just at startup.
|
||||
// When a resource is marked as "migrating", all write requests will be 503 unavailable
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestRuntime_Create(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, tt.input)
|
||||
}
|
||||
|
||||
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
|
||||
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
|
||||
require.NoError(t, err)
|
||||
dw, err := m.NewStorage(kind, ls, us)
|
||||
require.NoError(t, err)
|
||||
@@ -150,7 +150,7 @@ func TestRuntime_Get(t *testing.T) {
|
||||
tt.setupStorageFn(us.Mock, name)
|
||||
}
|
||||
|
||||
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
|
||||
m, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
|
||||
require.NoError(t, err)
|
||||
dw, err := m.NewStorage(kind, ls, us)
|
||||
require.NoError(t, err)
|
||||
@@ -235,7 +235,7 @@ func TestRuntime_CreateWhileMigrating(t *testing.T) {
|
||||
}
|
||||
|
||||
// Shared provider across all tests
|
||||
dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), nil)
|
||||
dual, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagManagedDualWriter), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -12,8 +12,28 @@ import (
|
||||
"github.com/grafana/grafana/pkg/infra/kvstore"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
|
||||
)
|
||||
|
||||
// fakeMigrator is a no-op implementation of UnifiedStorageMigrationService
|
||||
type fakeMigrator struct{}
|
||||
|
||||
func (f *fakeMigrator) Run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ unifiedmigrations.UnifiedStorageMigrationService = (*fakeMigrator)(nil)
|
||||
|
||||
func NewFakeMigrator() unifiedmigrations.UnifiedStorageMigrationService {
|
||||
return &fakeMigrator{}
|
||||
}
|
||||
|
||||
func NewFakeConfig() *setting.Cfg {
|
||||
return &setting.Cfg{
|
||||
UnifiedStorage: make(map[string]setting.UnifiedStorageConfig),
|
||||
}
|
||||
}
|
||||
|
||||
func ProvideStaticServiceForTests(cfg *setting.Cfg) Service {
|
||||
if cfg == nil {
|
||||
cfg = &setting.Cfg{}
|
||||
@@ -25,7 +45,14 @@ func ProvideService(
|
||||
features featuremgmt.FeatureToggles,
|
||||
kv kvstore.KVStore,
|
||||
cfg *setting.Cfg,
|
||||
migrator unifiedmigrations.UnifiedStorageMigrationService,
|
||||
) (Service, error) {
|
||||
// Ensure migrations have run before starting dualwrite
|
||||
err := migrator.Run(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to start dualwrite service due to migration error: %w", err)
|
||||
}
|
||||
|
||||
//nolint:staticcheck // not yet migrated to OpenFeature
|
||||
enabled := features.IsEnabledGlobally(featuremgmt.FlagManagedDualWriter) ||
|
||||
features.IsEnabledGlobally(featuremgmt.FlagProvisioning) // required for git provisioning
|
||||
@@ -64,6 +91,37 @@ type service struct {
|
||||
enabled bool
|
||||
}
|
||||
|
||||
func (m *service) NewStorage(gr schema.GroupResource, legacy rest.Storage, unified rest.Storage) (rest.Storage, error) {
|
||||
status, err := m.Status(context.Background(), gr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if m.enabled && status.Runtime {
|
||||
// Dynamic storage behavior
|
||||
return &runtimeDualWriter{
|
||||
service: m,
|
||||
legacy: legacy,
|
||||
unified: unified,
|
||||
dualwrite: &dualWriter{legacy: legacy, unified: unified}, // not used for read
|
||||
gr: gr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if status.ReadUnified {
|
||||
if status.WriteLegacy {
|
||||
// Write both, read unified
|
||||
return &dualWriter{legacy: legacy, unified: unified, readUnified: true}, nil
|
||||
}
|
||||
return unified, nil
|
||||
}
|
||||
if status.WriteUnified {
|
||||
// Write both, read legacy
|
||||
return &dualWriter{legacy: legacy, unified: unified}, nil
|
||||
}
|
||||
return legacy, nil
|
||||
}
|
||||
|
||||
// Hardcoded list of resources that should be controlled by the database (eventually everything?)
|
||||
func (m *service) ShouldManage(gr schema.GroupResource) bool {
|
||||
if !m.enabled {
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
func TestService(t *testing.T) {
|
||||
t.Run("dynamic", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mode, err := ProvideService(featuremgmt.WithFeatures(), kvstore.NewFakeKVStore(), nil)
|
||||
mode, err := ProvideService(featuremgmt.WithFeatures(featuremgmt.FlagProvisioning), kvstore.NewFakeKVStore(), NewFakeConfig(), NewFakeMigrator())
|
||||
require.NoError(t, err)
|
||||
|
||||
gr := schema.GroupResource{Group: "ggg", Resource: "rrr"}
|
||||
@@ -122,7 +122,7 @@ func TestService(t *testing.T) {
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg)
|
||||
svc, err := ProvideService(tc.flags, kvstore.NewFakeKVStore(), &tc.cfg, NewFakeMigrator())
|
||||
if tc.error != "" {
|
||||
require.ErrorContains(t, err, tc.error)
|
||||
require.Nil(t, svc, "expect a nil service when an error exts")
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
// NewDualWriter -- temporary shim
|
||||
func NewDualWriter(
|
||||
// NewStaticStorage -- temporary shim
|
||||
func NewStaticStorage(
|
||||
gr schema.GroupResource,
|
||||
mode rest.DualWriterMode,
|
||||
legacy rest.Storage,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package contract
|
||||
|
||||
import (
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
)
|
||||
|
||||
// UnifiedStorageMigrationService provides unified storage migrations as a background service.
|
||||
// This interface is defined in a separate package to avoid import cycles between
|
||||
// the migrations implementation and packages that need to depend on it (like dualwrite).
|
||||
type UnifiedStorageMigrationService interface {
|
||||
registry.BackgroundService
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
FoldersAndDashboardsMigrationID = "folders and dashboards migration"
|
||||
UnifiedStorageDataMigrationSQL = "unified storage data migration"
|
||||
)
|
||||
|
||||
type dashboardAndFolderMigration struct {
|
||||
migrator.MigrationBase
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
bulkStoreClient resource.ResourceClient
|
||||
}
|
||||
|
||||
var _ migrator.CodeMigration = (*dashboardAndFolderMigration)(nil)
|
||||
|
||||
// SQL implements migrator.Migration interface. Returns a description string.
|
||||
func (sp *dashboardAndFolderMigration) SQL(dialect migrator.Dialect) string {
|
||||
return UnifiedStorageDataMigrationSQL
|
||||
}
|
||||
|
||||
func (sp *dashboardAndFolderMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
|
||||
ctx := context.Background()
|
||||
logger := mg.Logger
|
||||
|
||||
resources := []schema.GroupResource{
|
||||
{
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
},
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
},
|
||||
}
|
||||
|
||||
storageMigrator := newUnifiedStorageMigrator(sp.legacyMigrator, sp.bulkStoreClient, resources, "unified-storage-migration.folders-dashboards")
|
||||
|
||||
orgs, err := sp.getAllOrgs(sess)
|
||||
if err != nil {
|
||||
logger.Error("failed to get organizations for folders and dashboards migration", "error", err)
|
||||
return fmt.Errorf("failed to get organizations: %w", err)
|
||||
}
|
||||
|
||||
if len(orgs) == 0 {
|
||||
logger.Info("No organizations found to migrate, skipping migration")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("Starting migration for all organizations", "org_count", len(orgs))
|
||||
|
||||
for _, org := range orgs {
|
||||
namespace := types.OrgNamespaceFormatter(org.ID)
|
||||
logger.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
|
||||
|
||||
// Create a service identity context for this namespace to authenticate with unified storage
|
||||
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
|
||||
|
||||
if err := storageMigrator.executeMigration(migrationCtx, sess, mg, namespace); err != nil {
|
||||
logger.Error("migration failed for organization", "org_id", org.ID, "org_name", org.Name, "error", err)
|
||||
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
|
||||
return nil
|
||||
}
|
||||
|
||||
type orgInfo struct {
|
||||
ID int64 `xorm:"id"`
|
||||
Name string `xorm:"name"`
|
||||
}
|
||||
|
||||
func (sp *dashboardAndFolderMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
|
||||
var orgs []orgInfo
|
||||
err := sess.Table("org").Cols("id", "name").Find(&orgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orgs, nil
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
|
||||
var logger = log.New("storage.unified.migrations")
|
||||
|
||||
// UnifiedStorageMigrationProvider provides unified storage migrations as a background service
|
||||
type UnifiedStorageMigrationProvider interface {
|
||||
registry.BackgroundService
|
||||
}
|
||||
|
||||
type UnifiedStorageMigrationProviderImpl struct {
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
cfg *setting.Cfg
|
||||
client resource.ResourceClient
|
||||
sqlStore db.DB
|
||||
}
|
||||
|
||||
var _ UnifiedStorageMigrationProvider = (*UnifiedStorageMigrationProviderImpl)(nil)
|
||||
|
||||
// ProvideUnifiedStorageMigrationProvider is a Wire provider that creates the migration service.
|
||||
// The service implements registry.BackgroundService and runs migrations during server startup.
|
||||
func ProvideUnifiedStorageMigrationProvider(
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
cfg *setting.Cfg,
|
||||
client resource.ResourceClient,
|
||||
sqlStore db.DB,
|
||||
) *UnifiedStorageMigrationProviderImpl {
|
||||
return &UnifiedStorageMigrationProviderImpl{
|
||||
legacyMigrator: legacyMigrator,
|
||||
cfg: cfg,
|
||||
client: client,
|
||||
sqlStore: sqlStore,
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes unified storage migrations as a background service.
|
||||
// This blocks until migrations complete. If migrations fail, an error is returned
|
||||
// which will prevent Grafana from starting.
|
||||
func (p *UnifiedStorageMigrationProviderImpl) Run(ctx context.Context) error {
|
||||
// skip migrations in test environments to prevent integration test timeouts.
|
||||
if os.Getenv("GRAFANA_TEST_DB") != "" {
|
||||
return nil
|
||||
}
|
||||
// skip migrations if disabled in config
|
||||
if p.cfg.DisableDataMigrations {
|
||||
logger.Info("Data migrations are disabled, skipping")
|
||||
return nil
|
||||
}
|
||||
// TODO: Re-enable once migrations are ready
|
||||
// return RegisterMigrations(p.legacyMigrator, p.cfg, p.client, p.sqlStore)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterMigrations initializes and registers all unified storage migrations.
|
||||
// This function is the entry point for all data migrations from legacy storage
|
||||
// to unified storage. It returns an error if migrations fail, preventing Grafana
|
||||
// from starting with inconsistent data.
|
||||
func RegisterMigrations(
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
cfg *setting.Cfg,
|
||||
client resource.ResourceClient,
|
||||
sqlStore db.DB,
|
||||
) error {
|
||||
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
|
||||
defer span.End()
|
||||
mg := migrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unified_storage")
|
||||
mg.AddCreateMigration()
|
||||
|
||||
if err := prometheus.Register(mg); err != nil {
|
||||
logger.Warn("Failed to register migrator metrics", "error", err)
|
||||
}
|
||||
|
||||
// Add new migration registrations here for each resource type
|
||||
registerDashboardAndFolderMigration(mg, legacyMigrator, client)
|
||||
|
||||
// Run all registered migrations (blocking)
|
||||
sec := cfg.Raw.Section("database")
|
||||
if err := mg.RunMigrations(ctx, sec.Key("migration_locking").MustBool(true), sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
|
||||
return fmt.Errorf("unified storage data migration failed: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("Unified storage migrations completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func registerDashboardAndFolderMigration(
|
||||
mg *migrator.Migrator,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
bulkStoreClient resource.ResourceClient,
|
||||
) {
|
||||
migration := &dashboardAndFolderMigration{
|
||||
legacyMigrator: legacyMigrator,
|
||||
bulkStoreClient: bulkStoreClient,
|
||||
}
|
||||
mg.AddMigration(FoldersAndDashboardsMigrationID, migration)
|
||||
}
|
||||
@@ -3,164 +3,200 @@ package migrations
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// StorageMigrator defines the interface for executing unified storage migrations
|
||||
type StorageMigrator interface {
|
||||
executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error
|
||||
// Read from legacy and write into unified storage
|
||||
//
|
||||
//go:generate mockery --name UnifiedMigrator --structname MockUnifiedMigrator --inpackage --filename migrator_mock.go --with-expecter
|
||||
type UnifiedMigrator interface {
|
||||
Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
}
|
||||
|
||||
type unifiedStorageMigrator struct {
|
||||
migrator legacy.LegacyMigrator
|
||||
bulkStoreClient resource.ResourceClient
|
||||
resources []schema.GroupResource
|
||||
log log.Logger
|
||||
// unifiedMigration handles the migration of legacy resources to unified storage
|
||||
type unifiedMigration struct {
|
||||
legacy.MigrationDashboardAccessor
|
||||
streamProvider streamProvider
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func newUnifiedStorageMigrator(migrator legacy.LegacyMigrator, bulkStoreClient resource.ResourceClient, resources []schema.GroupResource, logPrefix string) StorageMigrator {
|
||||
return &unifiedStorageMigrator{
|
||||
migrator: migrator,
|
||||
bulkStoreClient: bulkStoreClient,
|
||||
resources: resources,
|
||||
log: log.New(logPrefix),
|
||||
}
|
||||
// streamProvider abstracts the different ways to create a bulk process stream
|
||||
type streamProvider interface {
|
||||
createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error)
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) executeMigration(ctx context.Context, sess *xorm.Session, mg *migrator.Migrator, namespace string) error {
|
||||
startTime := time.Now()
|
||||
m.log.Info("Starting unified storage migration", "namespace", namespace, "resources", m.resources)
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Store: m.bulkStoreClient,
|
||||
LargeObjects: nil, // Not using large object support to avoid import cycles
|
||||
Resources: m.resources,
|
||||
WithHistory: true, // Migrate with full history
|
||||
OnlyCount: false,
|
||||
Progress: func(count int, msg string) {
|
||||
m.log.Info("Migration progress", "count", count, "message", msg)
|
||||
},
|
||||
}
|
||||
|
||||
// Execute the migration via legacy migrator
|
||||
response, err := m.migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
m.log.Error("Migration failed", "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("failed to migrate resources: %w", err)
|
||||
}
|
||||
|
||||
// Validate the migration results
|
||||
if err := m.validateMigration(sess, response); err != nil {
|
||||
m.log.Error("Migration validation failed", "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration validation failed: %w", err)
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed successfully",
|
||||
"duration", time.Since(startTime),
|
||||
"processed", response.Processed,
|
||||
"summaries", len(response.Summary),
|
||||
"rejected", len(response.Rejected))
|
||||
|
||||
return nil
|
||||
// resourceClientStreamProvider creates streams using resource.ResourceClient
|
||||
type resourceClientStreamProvider struct {
|
||||
client resource.ResourceClient
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
|
||||
// Check for rejected items
|
||||
if len(response.Rejected) > 0 {
|
||||
m.log.Warn("Migration had rejected items", "count", len(response.Rejected))
|
||||
for i, rejected := range response.Rejected {
|
||||
if i < 10 { // Log first 10 rejected items
|
||||
m.log.Warn("Rejected item",
|
||||
"namespace", rejected.Key.Namespace,
|
||||
"group", rejected.Key.Group,
|
||||
"resource", rejected.Key.Resource,
|
||||
"name", rejected.Key.Name,
|
||||
"reason", rejected.Error)
|
||||
}
|
||||
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
// Build collection settings for resource client
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.DASHBOARD_RESOURCE,
|
||||
})
|
||||
}
|
||||
// Rejections are not fatal - they may be expected for invalid data
|
||||
}
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
return r.client.BulkProcess(ctx)
|
||||
}
|
||||
|
||||
// bulkStoreClientStreamProvider creates streams using resourcepb.BulkStoreClient
|
||||
type bulkStoreClientStreamProvider struct {
|
||||
client resourcepb.BulkStoreClient
|
||||
}
|
||||
|
||||
func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
// Build collection settings for resource client
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: v1beta1.GROUP,
|
||||
Resource: v1beta1.DASHBOARD_RESOURCE,
|
||||
})
|
||||
}
|
||||
}
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
return b.client.BulkProcess(ctx)
|
||||
}
|
||||
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideUnifiedMigrator(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
client resource.ResourceClient,
|
||||
) UnifiedMigrator {
|
||||
return newUnifiedMigrator(
|
||||
dashboardAccess,
|
||||
&resourceClientStreamProvider{client: client},
|
||||
log.New("storage.unified.migrator"),
|
||||
)
|
||||
}
|
||||
|
||||
func ProvideUnifiedMigratorParquet(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
client resourcepb.BulkStoreClient,
|
||||
) UnifiedMigrator {
|
||||
return newUnifiedMigrator(
|
||||
dashboardAccess,
|
||||
&bulkStoreClientStreamProvider{client: client},
|
||||
log.New("storage.unified.migrator.parquet"),
|
||||
)
|
||||
}
|
||||
|
||||
func newUnifiedMigrator(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
streamProvider streamProvider,
|
||||
log log.Logger,
|
||||
) UnifiedMigrator {
|
||||
return &unifiedMigration{
|
||||
MigrationDashboardAccessor: dashboardAccess,
|
||||
streamProvider: streamProvider,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// migrate function -- works for a single kind
|
||||
type migratorFunc = func(ctx context.Context, orgId int64, opts legacy.MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*legacy.BlobStoreInfo, error)
|
||||
|
||||
func (m *unifiedMigration) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
info, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts.Progress == nil {
|
||||
opts.Progress = func(count int, msg string) {} // noop
|
||||
}
|
||||
|
||||
// Validate counts for each resource type
|
||||
for _, summary := range response.Summary {
|
||||
legacyCount, err := m.getLegacyCount(sess, summary.Group, summary.Resource, summary.Namespace)
|
||||
if len(opts.Resources) < 1 {
|
||||
return nil, fmt.Errorf("missing resource selector")
|
||||
}
|
||||
|
||||
if opts.OnlyCount {
|
||||
return m.CountResources(ctx, opts)
|
||||
}
|
||||
|
||||
stream, err := m.streamProvider.createStream(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
migratorFuncs := []migratorFunc{}
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateFolders)
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateLibraryPanels)
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
migratorFuncs = append(migratorFuncs, m.MigrateDashboards)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported resource: %s", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute migrations
|
||||
blobStore := legacy.BlobStoreInfo{}
|
||||
m.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
||||
for _, fn := range migratorFuncs {
|
||||
blobs, err := fn(ctx, info.OrgID, opts, stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get legacy count for %s/%s: %w", summary.Group, summary.Resource, err)
|
||||
m.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Account for rejected items in validation
|
||||
expectedCount := summary.Count + int64(len(response.Rejected))
|
||||
|
||||
m.log.Info("Count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace,
|
||||
"legacy_count", legacyCount,
|
||||
"unified_count", summary.Count,
|
||||
"rejected", len(response.Rejected),
|
||||
"history", summary.History)
|
||||
|
||||
// Validate that we migrated all items (allowing for rejected items)
|
||||
if legacyCount > expectedCount {
|
||||
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
|
||||
summary.Resource, summary.Group, summary.Namespace,
|
||||
legacyCount, summary.Count, len(response.Rejected))
|
||||
if blobs != nil {
|
||||
blobStore.Count += blobs.Count
|
||||
blobStore.Size += blobs.Size
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) getLegacyCount(sess *xorm.Session, group, resourceType, namespace string) (int64, error) {
|
||||
// Parse namespace to get org ID
|
||||
orgID, err := ParseOrgIDFromNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid namespace %s: %w", namespace, err)
|
||||
}
|
||||
|
||||
// Map group/resource to legacy table
|
||||
tableName, whereClause := m.getLegacyTableInfo(group, resourceType)
|
||||
if tableName == "" {
|
||||
return 0, fmt.Errorf("unknown resource type: %s.%s", resourceType, group)
|
||||
}
|
||||
|
||||
// Count items in legacy table using Table() before Count()
|
||||
count, err := sess.Table(tableName).Where(whereClause, orgID).Count()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count %s: %w", tableName, err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (m *unifiedStorageMigrator) getLegacyTableInfo(group, resource string) (table string, whereClause string) {
|
||||
// Map unified storage group/resource to legacy tables
|
||||
switch {
|
||||
case group == "dashboard.grafana.app" && resource == "dashboards":
|
||||
return "dashboard", "org_id = ? and is_folder = false"
|
||||
case group == "folder.grafana.app" && resource == "folders":
|
||||
return "dashboard", "org_id = ? and is_folder = true"
|
||||
case group == "playlist.grafana.app" && resource == "playlists":
|
||||
return "playlist", "org_id = ?"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
|
||||
// Use authlib to properly parse all namespace formats including "default" for org 1
|
||||
info, err := types.ParseNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse namespace: %w", err)
|
||||
}
|
||||
return info.OrgID, nil
|
||||
m.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
||||
return stream.CloseAndRecv()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
legacy "github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
// MockUnifiedMigrator is an autogenerated mock type for the UnifiedMigrator type
|
||||
type MockUnifiedMigrator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockUnifiedMigrator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockUnifiedMigrator) EXPECT() *MockUnifiedMigrator_Expecter {
|
||||
return &MockUnifiedMigrator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Migrate provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockUnifiedMigrator) Migrate(ctx context.Context, opts legacy.MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Migrate")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, legacy.MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, legacy.MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockUnifiedMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
|
||||
type MockUnifiedMigrator_Migrate_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Migrate is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts legacy.MigrateOptions
|
||||
func (_e *MockUnifiedMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockUnifiedMigrator_Migrate_Call {
|
||||
return &MockUnifiedMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) Run(run func(ctx context.Context, opts legacy.MigrateOptions)) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(legacy.MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockUnifiedMigrator_Migrate_Call) RunAndReturn(run func(context.Context, legacy.MigrateOptions) (*resourcepb.BulkResponse, error)) *MockUnifiedMigrator_Migrate_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockUnifiedMigrator creates a new instance of MockUnifiedMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockUnifiedMigrator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockUnifiedMigrator {
|
||||
mock := &MockUnifiedMigrator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/xorm"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// ValidationFunc is a function that validates migration results.
|
||||
// It receives the database session, migration response, and logger for reporting.
|
||||
// Return an error if validation fails, nil if validation passes or is skipped.
|
||||
type ValidationFunc func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error
|
||||
|
||||
// ResourceMigration handles migration of specific resource types from legacy to unified storage.
|
||||
// It implements migrator.CodeMigration and provides a generic, extensible way to migrate any
|
||||
// resource type by:
|
||||
//
|
||||
// 1. Iterating through all organizations
|
||||
// 2. For each org, delegating to LegacyMigrator to read from legacy and write to unified storage
|
||||
// 3. Validating migration results using the provided validation function (if any)
|
||||
//
|
||||
// To add a new resource type migration, simply create a new ResourceMigration instance in
|
||||
// service.go with the appropriate schema.GroupResource specifications and optional validation function.
|
||||
type ResourceMigration struct {
|
||||
migrator.MigrationBase
|
||||
migrator UnifiedMigrator
|
||||
resources []schema.GroupResource
|
||||
migrationID string
|
||||
validationFunc ValidationFunc // Optional: custom validation logic for this migration
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
// NewResourceMigration creates a new migration for the specified resources.
|
||||
// This is the primary way to register new resource migrations.
|
||||
//
|
||||
// Parameters:
|
||||
// - legacyMigrator: handles reading from legacy storage and writing to unified storage
|
||||
// - resources: list of GroupResource to migrate
|
||||
// - migrationID: unique identifier for this migration
|
||||
// - validationFunc: optional validation function to verify migration results.
|
||||
// If nil, no validation will be performed.
|
||||
//
|
||||
// Example with legacy table count validation:
|
||||
//
|
||||
// NewResourceMigration(
|
||||
// migrator,
|
||||
// []schema.GroupResource{{Group: "playlist.grafana.app", Resource: "playlists"}},
|
||||
// "playlists",
|
||||
// NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
// "playlist.grafana.app/playlists": {Table: "playlist", WhereClause: "org_id = ?"},
|
||||
// }),
|
||||
// )
|
||||
//
|
||||
// Example without validation:
|
||||
//
|
||||
// NewResourceMigration(migrator, resources, "new-resource", nil)
|
||||
func NewResourceMigration(
|
||||
migrator UnifiedMigrator,
|
||||
resources []schema.GroupResource,
|
||||
migrationID string,
|
||||
validationFunc ValidationFunc,
|
||||
) *ResourceMigration {
|
||||
return &ResourceMigration{
|
||||
migrator: migrator,
|
||||
resources: resources,
|
||||
migrationID: migrationID,
|
||||
validationFunc: validationFunc,
|
||||
log: log.New("storage.unified.resource_migration." + migrationID),
|
||||
}
|
||||
}
|
||||
|
||||
var _ migrator.CodeMigration = (*ResourceMigration)(nil)
|
||||
|
||||
// SQL implements migrator.Migration interface. Returns a description string.
|
||||
func (m *ResourceMigration) SQL(_ migrator.Dialect) string {
|
||||
return fmt.Sprintf("unified storage data migration: %s", m.migrationID)
|
||||
}
|
||||
|
||||
// Exec implements migrator.CodeMigration interface. Executes the migration across all organizations.
|
||||
func (m *ResourceMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
|
||||
ctx := context.Background()
|
||||
|
||||
orgs, err := m.getAllOrgs(sess)
|
||||
if err != nil {
|
||||
m.log.Error("failed to get organizations", "error", err)
|
||||
return fmt.Errorf("failed to get organizations: %w", err)
|
||||
}
|
||||
|
||||
if len(orgs) == 0 {
|
||||
m.log.Info("No organizations found to migrate, skipping migration")
|
||||
return nil
|
||||
}
|
||||
|
||||
m.log.Info("Starting migration for all organizations", "org_count", len(orgs), "resources", m.resources)
|
||||
|
||||
for _, org := range orgs {
|
||||
if err := m.migrateOrg(ctx, sess, org); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed successfully for all organizations", "org_count", len(orgs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateOrg handles migration for a single organization
|
||||
func (m *ResourceMigration) migrateOrg(ctx context.Context, sess *xorm.Session, org orgInfo) error {
|
||||
namespace := types.OrgNamespaceFormatter(org.ID)
|
||||
m.log.Info("Migrating organization", "org_id", org.ID, "org_name", org.Name, "namespace", namespace)
|
||||
|
||||
// Create a service identity context for this namespace to authenticate with unified storage
|
||||
migrationCtx, _ := identity.WithServiceIdentityForSingleNamespace(ctx, namespace)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Resources: m.resources,
|
||||
WithHistory: true, // Migrate with full history
|
||||
Progress: func(count int, msg string) {
|
||||
m.log.Info("Migration progress", "org_id", org.ID, "count", count, "message", msg)
|
||||
},
|
||||
}
|
||||
|
||||
// Execute the migration via legacy migrator
|
||||
response, err := m.migrator.Migrate(migrationCtx, opts)
|
||||
if err != nil {
|
||||
m.log.Error("Migration failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
|
||||
// Validate the migration results
|
||||
if err := m.validateMigration(sess, response); err != nil {
|
||||
m.log.Error("Migration validation failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
|
||||
return fmt.Errorf("migration validation failed for org %d (%s): %w", org.ID, org.Name, err)
|
||||
}
|
||||
|
||||
m.log.Info("Migration completed for organization",
|
||||
"org_id", org.ID,
|
||||
"duration", time.Since(startTime),
|
||||
"processed", response.Processed,
|
||||
"summaries", len(response.Summary),
|
||||
"rejected", len(response.Rejected))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateMigration calls the custom validation function if provided
|
||||
func (m *ResourceMigration) validateMigration(sess *xorm.Session, response *resourcepb.BulkResponse) error {
|
||||
if m.validationFunc == nil {
|
||||
m.log.Debug("No validation function provided, skipping validation")
|
||||
return nil
|
||||
}
|
||||
|
||||
return m.validationFunc(sess, response, m.log)
|
||||
}
|
||||
|
||||
// LegacyTableInfo defines how to map a unified storage resource to its legacy table
|
||||
type LegacyTableInfo struct {
|
||||
Table string // Legacy table name (e.g., "dashboard", "playlist")
|
||||
WhereClause string // WHERE clause template with org_id parameter (e.g., "org_id = ? and is_folder = false")
|
||||
}
|
||||
|
||||
// NewLegacyTableCountValidator creates a ValidationFunc that validates migration by comparing
|
||||
// counts between legacy tables and unified storage.
|
||||
//
|
||||
// This is a helper for the common case of validating that all items from legacy tables
|
||||
// were successfully migrated to unified storage.
|
||||
//
|
||||
// Parameters:
|
||||
// - legacyTableMap: maps "group/resource" keys to LegacyTableInfo for validation.
|
||||
// Only resources with mappings will be validated.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// validator := NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
// "dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
|
||||
// "folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
|
||||
// })
|
||||
func NewLegacyTableCountValidator(legacyTableMap map[string]LegacyTableInfo) ValidationFunc {
|
||||
return func(sess *xorm.Session, response *resourcepb.BulkResponse, log log.Logger) error {
|
||||
// Check for rejected items
|
||||
if len(response.Rejected) > 0 {
|
||||
log.Warn("Migration had rejected items", "count", len(response.Rejected))
|
||||
for i, rejected := range response.Rejected {
|
||||
if i < 10 { // Log first 10 rejected items
|
||||
log.Warn("Rejected item",
|
||||
"namespace", rejected.Key.Namespace,
|
||||
"group", rejected.Key.Group,
|
||||
"resource", rejected.Key.Resource,
|
||||
"name", rejected.Key.Name,
|
||||
"reason", rejected.Error)
|
||||
}
|
||||
}
|
||||
// Rejections are not fatal - they may be expected for invalid data
|
||||
}
|
||||
|
||||
// Validate counts for each resource type
|
||||
for _, summary := range response.Summary {
|
||||
key := fmt.Sprintf("%s/%s", summary.Group, summary.Resource)
|
||||
tableInfo, ok := legacyTableMap[key]
|
||||
if !ok {
|
||||
log.Debug("No legacy table mapping for resource, skipping count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace)
|
||||
continue
|
||||
}
|
||||
|
||||
// Get legacy count
|
||||
orgID, err := ParseOrgIDFromNamespace(summary.Namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid namespace %s: %w", summary.Namespace, err)
|
||||
}
|
||||
|
||||
legacyCount, err := sess.Table(tableInfo.Table).Where(tableInfo.WhereClause, orgID).Count()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to count %s: %w", tableInfo.Table, err)
|
||||
}
|
||||
|
||||
// Account for rejected items in validation
|
||||
expectedCount := summary.Count + int64(len(response.Rejected))
|
||||
|
||||
log.Info("Count validation",
|
||||
"resource", fmt.Sprintf("%s.%s", summary.Resource, summary.Group),
|
||||
"namespace", summary.Namespace,
|
||||
"legacy_count", legacyCount,
|
||||
"unified_count", summary.Count,
|
||||
"rejected", len(response.Rejected),
|
||||
"history", summary.History)
|
||||
|
||||
// Validate that we migrated all items (allowing for rejected items)
|
||||
if legacyCount > expectedCount {
|
||||
return fmt.Errorf("count mismatch for %s.%s in namespace %s: legacy has %d, unified has %d, rejected %d",
|
||||
summary.Resource, summary.Group, summary.Namespace,
|
||||
legacyCount, summary.Count, len(response.Rejected))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func ParseOrgIDFromNamespace(namespace string) (int64, error) {
|
||||
// Use authlib to properly parse all namespace formats including "default" for org 1
|
||||
info, err := types.ParseNamespace(namespace)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse namespace: %w", err)
|
||||
}
|
||||
return info.OrgID, nil
|
||||
}
|
||||
|
||||
// orgInfo represents basic organization information
|
||||
type orgInfo struct {
|
||||
ID int64 `xorm:"id"`
|
||||
Name string `xorm:"name"`
|
||||
}
|
||||
|
||||
// getAllOrgs retrieves all organizations from the database
|
||||
func (m *ResourceMigration) getAllOrgs(sess *xorm.Session) ([]orgInfo, error) {
|
||||
var orgs []orgInfo
|
||||
err := sess.Table("org").Cols("id", "name").Find(&orgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orgs, nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/kvstore"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/migrations")
|
||||
var logger = log.New("storage.unified.migrations")
|
||||
|
||||
type UnifiedStorageMigrationServiceImpl struct {
|
||||
migrator UnifiedMigrator
|
||||
cfg *setting.Cfg
|
||||
sqlStore db.DB
|
||||
kv kvstore.KVStore
|
||||
}
|
||||
|
||||
var _ contract.UnifiedStorageMigrationService = (*UnifiedStorageMigrationServiceImpl)(nil)
|
||||
|
||||
// ProvideUnifiedStorageMigrationService is a Wire provider that creates the migration service.
|
||||
// The service implements registry.BackgroundService and runs migrations during server startup.
|
||||
func ProvideUnifiedStorageMigrationService(
|
||||
migrator UnifiedMigrator,
|
||||
cfg *setting.Cfg,
|
||||
sqlStore db.DB,
|
||||
kv kvstore.KVStore,
|
||||
) contract.UnifiedStorageMigrationService {
|
||||
return &UnifiedStorageMigrationServiceImpl{
|
||||
migrator: migrator,
|
||||
cfg: cfg,
|
||||
sqlStore: sqlStore,
|
||||
kv: kv,
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes unified storage migrations as a background service.
|
||||
// This blocks until migrations complete. If migrations fail, an error is returned
|
||||
// which will prevent Grafana from starting.
|
||||
func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error {
|
||||
// TODO: temporary skip migrations in test environments to prevent integration test timeouts.
|
||||
if os.Getenv("GRAFANA_TEST_DB") != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// skip migrations if disabled in config
|
||||
if p.cfg.DisableDataMigrations {
|
||||
logger.Info("Data migrations are disabled, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: Re-enable once migrations are ready
|
||||
// TODO: add guarantee that this only runs once
|
||||
// return RegisterMigrations(p.migrator, p.cfg, p.sqlStore)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterMigrations initializes and registers all unified storage migrations.
|
||||
// This function is the entry point for all data migrations from legacy storage
|
||||
// to unified storage. It returns an error if migrations fail, preventing Grafana
|
||||
// from starting with inconsistent data.
|
||||
func RegisterMigrations(
|
||||
migrator UnifiedMigrator,
|
||||
cfg *setting.Cfg,
|
||||
sqlStore db.DB,
|
||||
) error {
|
||||
ctx, span := tracer.Start(context.Background(), "storage.unified.RegisterMigrations")
|
||||
defer span.End()
|
||||
mg := sqlstoremigrator.NewScopedMigrator(sqlStore.GetEngine(), cfg, "unifiedstorage")
|
||||
mg.AddCreateMigration()
|
||||
|
||||
if err := prometheus.Register(mg); err != nil {
|
||||
logger.Warn("Failed to register migrator metrics", "error", err)
|
||||
}
|
||||
|
||||
// Register resource migrations
|
||||
// To add a new resource type, simply add another migration here with the appropriate resources
|
||||
registerResourceMigrations(mg, migrator)
|
||||
|
||||
// Run all registered migrations (blocking)
|
||||
sec := cfg.Raw.Section("database")
|
||||
if err := mg.RunMigrations(ctx,
|
||||
sec.Key("migration_locking").MustBool(true),
|
||||
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
|
||||
return fmt.Errorf("unified storage data migration failed: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("Unified storage migrations completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerResourceMigrations registers all unified storage resource migrations.
|
||||
// Add new resource types here by creating additional ResourceMigration instances.
|
||||
func registerResourceMigrations(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator) {
|
||||
dashboardsAndFolders := NewResourceMigration(
|
||||
migrator,
|
||||
[]schema.GroupResource{
|
||||
{Group: "folder.grafana.app", Resource: "folders"},
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
},
|
||||
"folders-dashboards",
|
||||
NewLegacyTableCountValidator(map[string]LegacyTableInfo{
|
||||
"folder.grafana.app/folders": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = true"},
|
||||
"dashboard.grafana.app/dashboards": {Table: "dashboard", WhereClause: "org_id = ? and is_folder = false"},
|
||||
}),
|
||||
)
|
||||
mg.AddMigration("folders and dashboards migration", dashboardsAndFolders)
|
||||
}
|
||||
Reference in New Issue
Block a user