feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
Mustafa Sencer Özcan
co-authored by
Rafael Paulovic
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -1,96 +0,0 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockLegacyMigrator is an autogenerated mock type for the LegacyMigrator type
|
||||
type MockLegacyMigrator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockLegacyMigrator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockLegacyMigrator) EXPECT() *MockLegacyMigrator_Expecter {
|
||||
return &MockLegacyMigrator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Migrate provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockLegacyMigrator) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Migrate")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockLegacyMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
|
||||
type MockLegacyMigrator_Migrate_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Migrate is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockLegacyMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockLegacyMigrator_Migrate_Call {
|
||||
return &MockLegacyMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockLegacyMigrator creates a new instance of MockLegacyMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockLegacyMigrator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockLegacyMigrator {
|
||||
mock := &MockLegacyMigrator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -1,460 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"google.golang.org/grpc/metadata"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Store resourcepb.BulkStoreClient
|
||||
LargeObjects apistore.LargeObjectSupport
|
||||
BlobStore resourcepb.BlobStoreClient
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
// Read from legacy and write into unified storage
|
||||
//
|
||||
//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter
|
||||
type LegacyMigrator interface {
|
||||
Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
}
|
||||
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideLegacyMigrator(
|
||||
sql db.DB, // direct access to tables
|
||||
provisioning provisioning.ProvisioningService, // only needed for dashboard settings
|
||||
libraryPanelSvc librarypanels.Service,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) LegacyMigrator {
|
||||
dbp := legacysql.NewDatabaseProvider(sql)
|
||||
return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), dashboardPermissionSvc, accessControl, features)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
// migrate function -- works for a single kind
|
||||
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
|
||||
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
info, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts.Progress == nil {
|
||||
opts.Progress = func(count int, msg string) {} // noop
|
||||
}
|
||||
|
||||
// Migrate everything
|
||||
if len(opts.Resources) < 1 {
|
||||
return nil, fmt.Errorf("missing resource selector")
|
||||
}
|
||||
|
||||
migratorFuncs := []migratorFunc{}
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateFolders)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
migratorFuncs = append(migratorFuncs, a.migratePanels)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
})
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported resource: %s", res)
|
||||
}
|
||||
}
|
||||
if opts.OnlyCount {
|
||||
return a.countValues(ctx, opts)
|
||||
}
|
||||
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
if md, ok := metadata.FromOutgoingContext(ctx); ok {
|
||||
a.log.Debug("bulk grpc request metadata",
|
||||
"metadata", md,
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
} else {
|
||||
a.log.Debug("bulk grpc request, no metadata found",
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
}
|
||||
|
||||
stream, err := opts.Store.BulkProcess(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now run each migration
|
||||
blobStore := BlobStoreInfo{}
|
||||
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
||||
for _, m := range migratorFuncs {
|
||||
blobs, err := m(ctx, info.OrgID, opts, stream)
|
||||
if err != nil {
|
||||
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
||||
return nil, err
|
||||
}
|
||||
if blobs != nil {
|
||||
blobStore.Count += blobs.Count
|
||||
blobStore.Size += blobs.Size
|
||||
}
|
||||
}
|
||||
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
||||
return stream.CloseAndRecv()
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
large := opts.LargeObjects
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
// With large object support
|
||||
if large != nil && len(body) > large.Threshold() {
|
||||
obj, err := utils.MetaAccessor(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body)))
|
||||
err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// The smaller version (most of spec removed)
|
||||
req.Value, err = json.Marshal(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
blobs.Count++
|
||||
blobs.Size += int64(len(body))
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockMigrationDashboardAccessor is an autogenerated mock type for the MigrationDashboardAccessor type
|
||||
type MockMigrationDashboardAccessor struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockMigrationDashboardAccessor_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockMigrationDashboardAccessor) EXPECT() *MockMigrationDashboardAccessor_Expecter {
|
||||
return &MockMigrationDashboardAccessor_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// CountResources provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockMigrationDashboardAccessor) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CountResources")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_CountResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountResources'
|
||||
type MockMigrationDashboardAccessor_CountResources_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CountResources is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) CountResources(ctx interface{}, opts interface{}) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
return &MockMigrationDashboardAccessor_CountResources_Call{Call: _e.mock.On("CountResources", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateDashboards provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateDashboards")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateDashboards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateDashboards'
|
||||
type MockMigrationDashboardAccessor_MigrateDashboards_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateDashboards is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateDashboards(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateDashboards_Call{Call: _e.mock.On("MigrateDashboards", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateFolders provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateFolders")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateFolders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateFolders'
|
||||
type MockMigrationDashboardAccessor_MigrateFolders_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateFolders is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateFolders(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateFolders_Call{Call: _e.mock.On("MigrateFolders", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateLibraryPanels")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateLibraryPanels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateLibraryPanels'
|
||||
type MockMigrationDashboardAccessor_MigrateLibraryPanels_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateLibraryPanels(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateLibraryPanels_Call{Call: _e.mock.On("MigrateLibraryPanels", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockMigrationDashboardAccessor creates a new instance of MockMigrationDashboardAccessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockMigrationDashboardAccessor(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockMigrationDashboardAccessor {
|
||||
mock := &MockMigrationDashboardAccessor{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"go.opentelemetry.io/otel"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/utils/ptr"
|
||||
|
||||
claims "github.com/grafana/authlib/types"
|
||||
@@ -20,6 +23,7 @@ import (
|
||||
dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
|
||||
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
@@ -35,16 +39,30 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
var (
|
||||
_ DashboardAccess = (*dashboardSqlAccess)(nil)
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
type dashboardRow struct {
|
||||
// The numeric version for this dashboard
|
||||
RV int64
|
||||
@@ -63,8 +81,9 @@ type dashboardRow struct {
|
||||
type dashboardSqlAccess struct {
|
||||
sql legacysql.LegacyDatabaseProvider
|
||||
namespacer request.NamespaceMapper
|
||||
provisioning provisioning.ProvisioningService
|
||||
provisioning provisioning.StubProvisioningService
|
||||
|
||||
// TODO: consider enabling this by default for on-prem migrations
|
||||
invalidDashboardParseFallbackEnabled bool
|
||||
|
||||
// Use for writing (not reading)
|
||||
@@ -73,7 +92,7 @@ type dashboardSqlAccess struct {
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService
|
||||
|
||||
accessControl accesscontrol.AccessControl
|
||||
libraryPanelSvc librarypanels.Service
|
||||
libraryPanelSvc librarypanels.Service // only used for save dashboard
|
||||
|
||||
// Typically one... the server wrapper
|
||||
subscribers []chan *resource.WrittenEvent
|
||||
@@ -81,7 +100,27 @@ type dashboardSqlAccess struct {
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
// ProvideMigratorDashboardAccessor creates a DashboardAccess specifically for migration purposes.
|
||||
// This provider is used by Wire DI and only includes the minimal dependencies needed for migrations.
|
||||
func ProvideMigratorDashboardAccessor(
|
||||
sql legacysql.LegacyDatabaseProvider,
|
||||
provisioning provisioning.StubProvisioningService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) MigrationDashboardAccessor {
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
namespacer: claims.OrgNamespaceFormatter,
|
||||
dashStore: nil, // not needed for migration
|
||||
provisioning: provisioning,
|
||||
dashboardPermissionSvc: nil, // not needed for migration
|
||||
libraryPanelSvc: nil, // not needed for migration
|
||||
accessControl: accessControl,
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
|
||||
func NewDashboardSQLAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
namespacer request.NamespaceMapper,
|
||||
dashStore dashboards.Store,
|
||||
provisioning provisioning.ProvisioningService,
|
||||
@@ -90,7 +129,7 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) DashboardAccess {
|
||||
) *dashboardSqlAccess {
|
||||
dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter)
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
@@ -101,7 +140,6 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc: dashboardPermissionSvc,
|
||||
libraryPanelSvc: libraryPanelSvc,
|
||||
accessControl: accessControl,
|
||||
log: log.New("dashboard.legacysql"),
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
@@ -149,6 +187,290 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
|
||||
}, err
|
||||
}
|
||||
|
||||
// CountResources counts resources without migrating them
|
||||
func (a *dashboardSqlAccess) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := claims.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
// MigrateDashboards handles the dashboard migration logic
|
||||
func (a *dashboardSqlAccess) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboardV1.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// MigrateFolders handles the folder migration logic
|
||||
func (a *dashboardSqlAccess) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels handles the library panel migration logic
|
||||
func (a *dashboardSqlAccess) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var _ resource.ListIterator = (*rowsWrapper)(nil)
|
||||
|
||||
type rowsWrapper struct {
|
||||
|
||||
@@ -21,6 +21,9 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/dashboards"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestScanRow(t *testing.T) {
|
||||
@@ -529,3 +532,146 @@ func TestParseLibraryPanelRow(t *testing.T) {
|
||||
require.Nil(t, updatedTimestamp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -55,10 +55,9 @@ type LibraryPanelQuery struct {
|
||||
LastID int64
|
||||
}
|
||||
|
||||
type DashboardAccess interface {
|
||||
type DashboardAccessor interface {
|
||||
resource.StorageBackend
|
||||
resourcepb.ResourceIndexServer
|
||||
LegacyMigrator
|
||||
|
||||
GetDashboard(ctx context.Context, orgId int64, uid string, version int64) (*dashboardV1.Dashboard, int64, error)
|
||||
SaveDashboard(ctx context.Context, orgId int64, dash *dashboardV1.Dashboard, failOnExisting bool) (*dashboardV1.Dashboard, bool, error)
|
||||
@@ -67,3 +66,12 @@ type DashboardAccess interface {
|
||||
// Get a typed list
|
||||
GetLibraryPanels(ctx context.Context, query LibraryPanelQuery) (*dashboardV0.LibraryPanelList, error)
|
||||
}
|
||||
|
||||
//go:generate mockery --name MigrationDashboardAccessor --structname MockMigrationDashboardAccessor --inpackage --filename migration_dashboard_accessor_mock.go --with-expecter
|
||||
type MigrationDashboardAccessor interface {
|
||||
// Migration helper methods - these support the separate LegacyMigrator
|
||||
CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
type DashboardStorage struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
DashboardService dashboards.DashboardService
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ var (
|
||||
)
|
||||
|
||||
type LibraryPanelStore struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
ResourceInfo utils.ResourceInfo
|
||||
service libraryelements.Service
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ func RegisterAPIService(
|
||||
publicDashboardService: publicDashboardService,
|
||||
|
||||
legacy: &DashboardStorage{
|
||||
Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
Access: legacy.NewDashboardSQLAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
DashboardService: dashboardService,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ type dtoBuilder = func(dashboard runtime.Object, access *dashboard.DashboardAcce
|
||||
// The DTO returns everything the UI needs in a single request
|
||||
type DTOConnector struct {
|
||||
getter rest.Getter
|
||||
legacy legacy.DashboardAccess
|
||||
legacy legacy.DashboardAccessor
|
||||
unified resource.ResourceClient
|
||||
largeObjects apistore.LargeObjectSupport
|
||||
accessControl accesscontrol.AccessControl
|
||||
@@ -42,7 +42,7 @@ type DTOConnector struct {
|
||||
func NewDTOConnector(
|
||||
getter rest.Getter,
|
||||
largeObjects apistore.LargeObjectSupport,
|
||||
legacyAccess legacy.DashboardAccess,
|
||||
legacyAccess legacy.DashboardAccessor,
|
||||
resourceClient resource.ResourceClient,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
scheme *runtime.Scheme,
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources/signature"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/parquet"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
@@ -29,7 +30,7 @@ type LegacyResourcesMigrator interface {
|
||||
type legacyResourcesMigrator struct {
|
||||
repositoryResources resources.RepositoryResourcesFactory
|
||||
parsers resources.ParserFactory
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
signerFactory signature.SignerFactory
|
||||
clients resources.ClientFactory
|
||||
exportFn export.ExportFn
|
||||
@@ -38,7 +39,7 @@ type legacyResourcesMigrator struct {
|
||||
func NewLegacyResourcesMigrator(
|
||||
repositoryResources resources.RepositoryResourcesFactory,
|
||||
parsers resources.ParserFactory,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
signerFactory signature.SignerFactory,
|
||||
clients resources.ClientFactory,
|
||||
exportFn export.ExportFn,
|
||||
@@ -46,7 +47,7 @@ func NewLegacyResourcesMigrator(
|
||||
return &legacyResourcesMigrator{
|
||||
repositoryResources: repositoryResources,
|
||||
parsers: parsers,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
signerFactory: signerFactory,
|
||||
clients: clients,
|
||||
exportFn: exportFn,
|
||||
@@ -94,7 +95,7 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
|
||||
reader := newLegacyResourceMigrator(
|
||||
rw,
|
||||
m.legacyMigrator,
|
||||
m.dashboardAccess,
|
||||
parser,
|
||||
repositoryResources,
|
||||
progress,
|
||||
@@ -113,21 +114,21 @@ func (m *legacyResourcesMigrator) Migrate(ctx context.Context, rw repository.Rea
|
||||
}
|
||||
|
||||
type legacyResourceResourceMigrator struct {
|
||||
repo repository.ReaderWriter
|
||||
legacy legacy.LegacyMigrator
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
repo repository.ReaderWriter
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
parser resources.Parser
|
||||
progress jobs.JobProgressRecorder
|
||||
namespace string
|
||||
kind schema.GroupResource
|
||||
options provisioning.MigrateJobOptions
|
||||
resources resources.RepositoryResources
|
||||
signer signature.Signer
|
||||
history map[string]string // UID >> file path
|
||||
}
|
||||
|
||||
func newLegacyResourceMigrator(
|
||||
repo repository.ReaderWriter,
|
||||
legacy legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
parser resources.Parser,
|
||||
resources resources.RepositoryResources,
|
||||
progress jobs.JobProgressRecorder,
|
||||
@@ -141,16 +142,16 @@ func newLegacyResourceMigrator(
|
||||
history = make(map[string]string)
|
||||
}
|
||||
return &legacyResourceResourceMigrator{
|
||||
repo: repo,
|
||||
legacy: legacy,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
repo: repo,
|
||||
dashboardAccess: dashboardAccess,
|
||||
parser: parser,
|
||||
progress: progress,
|
||||
options: options,
|
||||
namespace: namespace,
|
||||
kind: kind,
|
||||
resources: resources,
|
||||
signer: signer,
|
||||
history: history,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,14 +226,21 @@ func (r *legacyResourceResourceMigrator) Write(ctx context.Context, key *resourc
|
||||
|
||||
func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
r.progress.SetMessage(ctx, fmt.Sprintf("migrate %s resource", r.kind.Resource))
|
||||
|
||||
// Create a parquet migrator with this instance as the BulkResourceWriter
|
||||
parquetClient := parquet.NewBulkResourceWriterClient(r)
|
||||
migrator := unifiedmigrations.ProvideUnifiedMigratorParquet(
|
||||
r.dashboardAccess,
|
||||
parquetClient,
|
||||
)
|
||||
|
||||
opts := legacy.MigrateOptions{
|
||||
Namespace: r.namespace,
|
||||
WithHistory: r.options.History,
|
||||
Resources: []schema.GroupResource{r.kind},
|
||||
Store: parquet.NewBulkResourceWriterClient(r),
|
||||
OnlyCount: true, // first get the count
|
||||
}
|
||||
stats, err := r.legacy.Migrate(ctx, opts)
|
||||
stats, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to count legacy items %w", err)
|
||||
}
|
||||
@@ -248,7 +256,7 @@ func (r *legacyResourceResourceMigrator) Migrate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
opts.OnlyCount = false // this time actually write
|
||||
_, err = r.legacy.Migrate(ctx, opts)
|
||||
_, err = migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate legacy %s: %w", r.kind.Resource, err)
|
||||
}
|
||||
|
||||
@@ -93,8 +93,8 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
mockRepoResourcesFactory.On("Client", mock.Anything, mock.Anything).
|
||||
Return(mockRepoResources, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("legacy migrator error"))
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
signerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -136,7 +136,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
@@ -308,22 +308,18 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
History: true,
|
||||
}).Return(mockSigner, nil)
|
||||
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
// Mock CountResources for the count phase
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once()
|
||||
// Mock MigrateDashboards for the actual migration phase (dashboards resource)
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Count: 10,
|
||||
History: 5,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Migration phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{
|
||||
Count: 10,
|
||||
Size: 5,
|
||||
}, nil).Once()
|
||||
|
||||
mockClients := resources.NewMockResourceClients(t)
|
||||
mockClientFactory := resources.NewMockClientFactory(t)
|
||||
@@ -339,7 +335,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
migrator := NewLegacyResourcesMigrator(
|
||||
mockRepoResourcesFactory,
|
||||
mockParserFactory,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
mockSignerFactory,
|
||||
mockClientFactory,
|
||||
mockExportFn.Execute,
|
||||
@@ -362,7 +358,7 @@ func TestLegacyResourcesMigrator_Migrate(t *testing.T) {
|
||||
|
||||
mockParserFactory.AssertExpectations(t)
|
||||
mockRepoResourcesFactory.AssertExpectations(t)
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
mockClientFactory.AssertExpectations(t)
|
||||
mockExportFn.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
@@ -742,8 +738,8 @@ func TestLegacyResourceResourceMigrator_Write(t *testing.T) {
|
||||
|
||||
func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
t.Run("should fail when legacy migrate count fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("count error"))
|
||||
|
||||
@@ -752,7 +748,7 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
@@ -766,89 +762,91 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to count legacy items")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should fail when legacy migrate write fails", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
// For test-resources GroupResource, we don't know which method it will call, but since it's not dashboards/folders/librarypanels,
|
||||
// the Migrate will fail trying to map the resource type. Let's make it dashboards for this test.
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, errors.New("write error")).Once() // Write phase
|
||||
}), mock.Anything).Return(nil, errors.New("write error")).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "test-resources"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "migrate legacy test-resources: write error")
|
||||
require.Contains(t, err.Error(), "migrate legacy dashboards: write error")
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("should successfully migrate resource", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to history if history is greater than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 1,
|
||||
History: 100,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -856,39 +854,39 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signature.NewGrafanaSigner(),
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
t.Run("should set total to count if history is less than count", func(t *testing.T) {
|
||||
mockLegacyMigrator := legacy.NewMockLegacyMigrator(t)
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess := legacy.NewMockMigrationDashboardAccessor(t)
|
||||
mockDashboardAccess.On("CountResources", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{
|
||||
Summary: []*resourcepb.BulkResponse_Summary{
|
||||
{
|
||||
Group: "test.grafana.app",
|
||||
Resource: "tests",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Count: 200,
|
||||
History: 1,
|
||||
},
|
||||
},
|
||||
}, nil).Once() // Count phase
|
||||
mockLegacyMigrator.On("Migrate", mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
mockDashboardAccess.On("MigrateDashboards", mock.Anything, mock.Anything, mock.MatchedBy(func(opts legacy.MigrateOptions) bool {
|
||||
return !opts.OnlyCount && opts.Namespace == "test-namespace"
|
||||
})).Return(&resourcepb.BulkResponse{}, nil).Once() // Write phase
|
||||
}), mock.Anything).Return(&legacy.BlobStoreInfo{}, nil).Once() // Write phase
|
||||
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
progress.On("SetMessage", mock.Anything, mock.Anything).Return()
|
||||
@@ -897,20 +895,20 @@ func TestLegacyResourceResourceMigrator_Migrate(t *testing.T) {
|
||||
|
||||
migrator := newLegacyResourceMigrator(
|
||||
nil,
|
||||
mockLegacyMigrator,
|
||||
mockDashboardAccess,
|
||||
nil,
|
||||
nil,
|
||||
progress,
|
||||
provisioning.MigrateJobOptions{},
|
||||
"test-namespace",
|
||||
schema.GroupResource{Group: "test.grafana.app", Resource: "tests"},
|
||||
schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
signer,
|
||||
)
|
||||
|
||||
err := migrator.Migrate(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
mockLegacyMigrator.AssertExpectations(t)
|
||||
mockDashboardAccess.AssertExpectations(t)
|
||||
progress.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
@@ -109,7 +110,7 @@ type APIBuilder struct {
|
||||
jobHistoryConfig *JobHistoryConfig
|
||||
jobHistoryLoki *jobs.LokiJobHistory
|
||||
resourceLister resources.ResourceLister
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
dashboardAccess legacy.MigrationDashboardAccessor
|
||||
storageStatus dualwrite.Service
|
||||
unified resource.ResourceClient
|
||||
repoFactory repository.Factory
|
||||
@@ -135,7 +136,7 @@ func NewAPIBuilder(
|
||||
features featuremgmt.FeatureToggles,
|
||||
unified resource.ResourceClient,
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
access authlib.AccessChecker,
|
||||
@@ -158,6 +159,7 @@ func NewAPIBuilder(
|
||||
clients = resources.NewClientFactory(configProvider)
|
||||
}
|
||||
parsers := resources.NewParserFactory(clients)
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(dashboardAccess, unified)
|
||||
resourceLister := resources.NewResourceListerForMigrations(unified, legacyMigrator, storageStatus)
|
||||
|
||||
b := &APIBuilder{
|
||||
@@ -170,7 +172,7 @@ func NewAPIBuilder(
|
||||
parsers: parsers,
|
||||
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
|
||||
resourceLister: resourceLister,
|
||||
legacyMigrator: legacyMigrator,
|
||||
dashboardAccess: dashboardAccess,
|
||||
storageStatus: storageStatus,
|
||||
unified: unified,
|
||||
access: access,
|
||||
@@ -234,7 +236,7 @@ func RegisterAPIService(
|
||||
client resource.ResourceClient, // implements resource.RepositoryClient
|
||||
configProvider apiserver.RestConfigProvider,
|
||||
access authlib.AccessClient,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
storageStatus dualwrite.Service,
|
||||
usageStats usagestats.Service,
|
||||
tracer tracing.Tracer,
|
||||
@@ -258,7 +260,7 @@ func RegisterAPIService(
|
||||
features,
|
||||
client,
|
||||
configProvider,
|
||||
legacyMigrator, storageStatus,
|
||||
dashboardAccess, storageStatus,
|
||||
usageStats,
|
||||
access,
|
||||
tracer,
|
||||
@@ -722,7 +724,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
|
||||
legacyResources := migrate.NewLegacyResourcesMigrator(
|
||||
b.repositoryResources,
|
||||
b.parsers,
|
||||
b.legacyMigrator,
|
||||
b.dashboardAccess,
|
||||
signerFactory,
|
||||
b.clients,
|
||||
export.ExportAll,
|
||||
@@ -1241,8 +1243,9 @@ func (b *APIBuilder) tryRunningOnlyUnifiedStorage() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Count how many things exist
|
||||
rsp, err := b.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
// Count how many things exist - create a migrator on-demand for this
|
||||
legacyMigrator := migrations.ProvideUnifiedMigrator(b.dashboardAccess, b.unified)
|
||||
rsp, err := legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: "default", // FIXME! this works for single org, but need to check multi-org
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
@@ -30,9 +31,9 @@ type ResourceStore interface {
|
||||
}
|
||||
|
||||
type ResourceListerFromSearch struct {
|
||||
store ResourceStore
|
||||
legacyMigrator legacy.LegacyMigrator
|
||||
storageStatus dualwrite.Service
|
||||
store ResourceStore
|
||||
migrator migrations.UnifiedMigrator
|
||||
storageStatus dualwrite.Service
|
||||
}
|
||||
|
||||
func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
@@ -42,13 +43,13 @@ func NewResourceLister(store ResourceStore) ResourceLister {
|
||||
// FIXME: the logic about migration and storage should probably be separated from this
|
||||
func NewResourceListerForMigrations(
|
||||
store ResourceStore,
|
||||
legacyMigrator legacy.LegacyMigrator,
|
||||
migrator migrations.UnifiedMigrator,
|
||||
storageStatus dualwrite.Service,
|
||||
) ResourceLister {
|
||||
return &ResourceListerFromSearch{
|
||||
store: store,
|
||||
legacyMigrator: legacyMigrator,
|
||||
storageStatus: storageStatus,
|
||||
store: store,
|
||||
migrator: migrator,
|
||||
storageStatus: storageStatus,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,8 +134,8 @@ func (o *ResourceListerFromSearch) Stats(ctx context.Context, namespace, reposit
|
||||
}
|
||||
|
||||
// Get the stats based on what a migration could support
|
||||
if o.storageStatus != nil && o.legacyMigrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.legacyMigrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
if o.storageStatus != nil && o.migrator != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, o.storageStatus) {
|
||||
rsp, err := o.migrator.Migrate(ctx, legacy.MigrateOptions{
|
||||
Namespace: namespace,
|
||||
Resources: []schema.GroupResource{{
|
||||
Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
|
||||
@@ -48,7 +48,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl"
|
||||
"github.com/grafana/grafana/pkg/services/team/teamapi"
|
||||
"github.com/grafana/grafana/pkg/services/updatemanager"
|
||||
unifiedmigrations "github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
)
|
||||
|
||||
func ProvideBackgroundServiceRegistry(
|
||||
@@ -74,7 +73,6 @@ func ProvideBackgroundServiceRegistry(
|
||||
dashboardServiceImpl *service.DashboardServiceImpl,
|
||||
secretsGarbageCollectionWorker *secretsgarbagecollectionworker.Worker,
|
||||
fixedRolesLoader *accesscontrol.FixedRolesLoader,
|
||||
unifiedStorageMigrationProvider unifiedmigrations.UnifiedStorageMigrationProvider,
|
||||
// Need to make sure these are initialized, is there a better place to put them?
|
||||
_ dashboardsnapshots.Service,
|
||||
_ serviceaccounts.Service,
|
||||
@@ -91,7 +89,6 @@ func ProvideBackgroundServiceRegistry(
|
||||
notifications,
|
||||
rendering,
|
||||
tokenService,
|
||||
unifiedStorageMigrationProvider,
|
||||
provisioning,
|
||||
grafanaUpdateChecker,
|
||||
pluginsUpdateChecker,
|
||||
|
||||
Reference in New Issue
Block a user