feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
Mustafa Sencer Özcan
co-authored by
Rafael Paulovic
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -1,96 +0,0 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockLegacyMigrator is an autogenerated mock type for the LegacyMigrator type
|
||||
type MockLegacyMigrator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockLegacyMigrator_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockLegacyMigrator) EXPECT() *MockLegacyMigrator_Expecter {
|
||||
return &MockLegacyMigrator_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Migrate provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockLegacyMigrator) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Migrate")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockLegacyMigrator_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate'
|
||||
type MockLegacyMigrator_Migrate_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Migrate is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockLegacyMigrator_Expecter) Migrate(ctx interface{}, opts interface{}) *MockLegacyMigrator_Migrate_Call {
|
||||
return &MockLegacyMigrator_Migrate_Call{Call: _e.mock.On("Migrate", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLegacyMigrator_Migrate_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockLegacyMigrator_Migrate_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockLegacyMigrator creates a new instance of MockLegacyMigrator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockLegacyMigrator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockLegacyMigrator {
|
||||
mock := &MockLegacyMigrator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -1,460 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"google.golang.org/grpc/metadata"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Store resourcepb.BulkStoreClient
|
||||
LargeObjects apistore.LargeObjectSupport
|
||||
BlobStore resourcepb.BlobStoreClient
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
// Read from legacy and write into unified storage
|
||||
//
|
||||
//go:generate mockery --name LegacyMigrator --structname MockLegacyMigrator --inpackage --filename legacy_migrator_mock.go --with-expecter
|
||||
type LegacyMigrator interface {
|
||||
Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
}
|
||||
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideLegacyMigrator(
|
||||
sql db.DB, // direct access to tables
|
||||
provisioning provisioning.ProvisioningService, // only needed for dashboard settings
|
||||
libraryPanelSvc librarypanels.Service,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) LegacyMigrator {
|
||||
dbp := legacysql.NewDatabaseProvider(sql)
|
||||
return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), dashboardPermissionSvc, accessControl, features)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
// migrate function -- works for a single kind
|
||||
type migratorFunc = func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
|
||||
func (a *dashboardSqlAccess) Migrate(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
info, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opts.Progress == nil {
|
||||
opts.Progress = func(count int, msg string) {} // noop
|
||||
}
|
||||
|
||||
// Migrate everything
|
||||
if len(opts.Resources) < 1 {
|
||||
return nil, fmt.Errorf("missing resource selector")
|
||||
}
|
||||
|
||||
migratorFuncs := []migratorFunc{}
|
||||
settings := resource.BulkSettings{
|
||||
RebuildCollection: true,
|
||||
SkipValidation: true,
|
||||
}
|
||||
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateFolders)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: folders.GROUP,
|
||||
Resource: folders.RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
migratorFuncs = append(migratorFuncs, a.migratePanels)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
})
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
migratorFuncs = append(migratorFuncs, a.migrateDashboards)
|
||||
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
})
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported resource: %s", res)
|
||||
}
|
||||
}
|
||||
if opts.OnlyCount {
|
||||
return a.countValues(ctx, opts)
|
||||
}
|
||||
|
||||
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
|
||||
if md, ok := metadata.FromOutgoingContext(ctx); ok {
|
||||
a.log.Debug("bulk grpc request metadata",
|
||||
"metadata", md,
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
} else {
|
||||
a.log.Debug("bulk grpc request, no metadata found",
|
||||
"collection", settings.Collection,
|
||||
)
|
||||
}
|
||||
|
||||
stream, err := opts.Store.BulkProcess(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now run each migration
|
||||
blobStore := BlobStoreInfo{}
|
||||
a.log.Info("start migrating legacy resources", "namespace", opts.Namespace, "orgId", info.OrgID, "stackId", info.StackID)
|
||||
for _, m := range migratorFuncs {
|
||||
blobs, err := m(ctx, info.OrgID, opts, stream)
|
||||
if err != nil {
|
||||
a.log.Error("error migrating legacy resources", "error", err, "namespace", opts.Namespace)
|
||||
return nil, err
|
||||
}
|
||||
if blobs != nil {
|
||||
blobStore.Count += blobs.Count
|
||||
blobStore.Size += blobs.Size
|
||||
}
|
||||
}
|
||||
a.log.Info("finished migrating legacy resources", "blobStore", blobStore)
|
||||
return stream.CloseAndRecv()
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) countValues(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := authlib.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboard.GROUP
|
||||
summary.Resource = dashboard.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
large := opts.LargeObjects
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboard.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
// With large object support
|
||||
if large != nil && len(body) > large.Threshold() {
|
||||
obj, err := utils.MetaAccessor(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s Large object (%d)", dash.Generation, dash.Name, len(body)))
|
||||
err = large.Deconstruct(ctx, req.Key, opts.BlobStore, obj, req.Value)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// The smaller version (most of spec removed)
|
||||
req.Value, err = json.Marshal(dash)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
blobs.Count++
|
||||
blobs.Size += int64(len(body))
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) migratePanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboard.GROUP,
|
||||
Resource: dashboard.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockMigrationDashboardAccessor is an autogenerated mock type for the MigrationDashboardAccessor type
|
||||
type MockMigrationDashboardAccessor struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockMigrationDashboardAccessor_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockMigrationDashboardAccessor) EXPECT() *MockMigrationDashboardAccessor_Expecter {
|
||||
return &MockMigrationDashboardAccessor_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// CountResources provides a mock function with given fields: ctx, opts
|
||||
func (_m *MockMigrationDashboardAccessor) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CountResources")
|
||||
}
|
||||
|
||||
var r0 *resourcepb.BulkResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, MigrateOptions) *resourcepb.BulkResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*resourcepb.BulkResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, MigrateOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_CountResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CountResources'
|
||||
type MockMigrationDashboardAccessor_CountResources_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CountResources is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - opts MigrateOptions
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) CountResources(ctx interface{}, opts interface{}) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
return &MockMigrationDashboardAccessor_CountResources_Call{Call: _e.mock.On("CountResources", ctx, opts)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Run(run func(ctx context.Context, opts MigrateOptions)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(MigrateOptions))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) Return(_a0 *resourcepb.BulkResponse, _a1 error) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_CountResources_Call) RunAndReturn(run func(context.Context, MigrateOptions) (*resourcepb.BulkResponse, error)) *MockMigrationDashboardAccessor_CountResources_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateDashboards provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateDashboards")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateDashboards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateDashboards'
|
||||
type MockMigrationDashboardAccessor_MigrateDashboards_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateDashboards is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateDashboards(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateDashboards_Call{Call: _e.mock.On("MigrateDashboards", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateDashboards_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateDashboards_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateFolders provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateFolders")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateFolders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateFolders'
|
||||
type MockMigrationDashboardAccessor_MigrateFolders_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateFolders is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateFolders(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateFolders_Call{Call: _e.mock.On("MigrateFolders", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateFolders_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateFolders_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels provides a mock function with given fields: ctx, orgId, opts, stream
|
||||
func (_m *MockMigrationDashboardAccessor) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
ret := _m.Called(ctx, orgId, opts, stream)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MigrateLibraryPanels")
|
||||
}
|
||||
|
||||
var r0 *BlobStoreInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)); ok {
|
||||
return rf(ctx, orgId, opts, stream)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) *BlobStoreInfo); ok {
|
||||
r0 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*BlobStoreInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) error); ok {
|
||||
r1 = rf(ctx, orgId, opts, stream)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMigrationDashboardAccessor_MigrateLibraryPanels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MigrateLibraryPanels'
|
||||
type MockMigrationDashboardAccessor_MigrateLibraryPanels_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgId int64
|
||||
// - opts MigrateOptions
|
||||
// - stream resourcepb.BulkStore_BulkProcessClient
|
||||
func (_e *MockMigrationDashboardAccessor_Expecter) MigrateLibraryPanels(ctx interface{}, orgId interface{}, opts interface{}, stream interface{}) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
return &MockMigrationDashboardAccessor_MigrateLibraryPanels_Call{Call: _e.mock.On("MigrateLibraryPanels", ctx, orgId, opts, stream)}
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Run(run func(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(MigrateOptions), args[3].(resourcepb.BulkStore_BulkProcessClient))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) Return(_a0 *BlobStoreInfo, _a1 error) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call) RunAndReturn(run func(context.Context, int64, MigrateOptions, resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)) *MockMigrationDashboardAccessor_MigrateLibraryPanels_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockMigrationDashboardAccessor creates a new instance of MockMigrationDashboardAccessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockMigrationDashboardAccessor(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockMigrationDashboardAccessor {
|
||||
mock := &MockMigrationDashboardAccessor{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"go.opentelemetry.io/otel"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/utils/ptr"
|
||||
|
||||
claims "github.com/grafana/authlib/types"
|
||||
@@ -20,6 +23,7 @@ import (
|
||||
dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
|
||||
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
|
||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
@@ -35,16 +39,30 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/librarypanels"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
var (
|
||||
_ DashboardAccess = (*dashboardSqlAccess)(nil)
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
tracer = otel.Tracer("github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy")
|
||||
)
|
||||
|
||||
type MigrateOptions struct {
|
||||
Namespace string
|
||||
Resources []schema.GroupResource
|
||||
WithHistory bool // only applies to dashboards
|
||||
OnlyCount bool // just count the values
|
||||
Progress func(count int, msg string)
|
||||
}
|
||||
|
||||
type BlobStoreInfo struct {
|
||||
Count int64
|
||||
Size int64
|
||||
}
|
||||
|
||||
type dashboardRow struct {
|
||||
// The numeric version for this dashboard
|
||||
RV int64
|
||||
@@ -63,8 +81,9 @@ type dashboardRow struct {
|
||||
type dashboardSqlAccess struct {
|
||||
sql legacysql.LegacyDatabaseProvider
|
||||
namespacer request.NamespaceMapper
|
||||
provisioning provisioning.ProvisioningService
|
||||
provisioning provisioning.StubProvisioningService
|
||||
|
||||
// TODO: consider enabling this by default for on-prem migrations
|
||||
invalidDashboardParseFallbackEnabled bool
|
||||
|
||||
// Use for writing (not reading)
|
||||
@@ -73,7 +92,7 @@ type dashboardSqlAccess struct {
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService
|
||||
|
||||
accessControl accesscontrol.AccessControl
|
||||
libraryPanelSvc librarypanels.Service
|
||||
libraryPanelSvc librarypanels.Service // only used for save dashboard
|
||||
|
||||
// Typically one... the server wrapper
|
||||
subscribers []chan *resource.WrittenEvent
|
||||
@@ -81,7 +100,27 @@ type dashboardSqlAccess struct {
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
// ProvideMigratorDashboardAccessor creates a DashboardAccess specifically for migration purposes.
|
||||
// This provider is used by Wire DI and only includes the minimal dependencies needed for migrations.
|
||||
func ProvideMigratorDashboardAccessor(
|
||||
sql legacysql.LegacyDatabaseProvider,
|
||||
provisioning provisioning.StubProvisioningService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) MigrationDashboardAccessor {
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
namespacer: claims.OrgNamespaceFormatter,
|
||||
dashStore: nil, // not needed for migration
|
||||
provisioning: provisioning,
|
||||
dashboardPermissionSvc: nil, // not needed for migration
|
||||
libraryPanelSvc: nil, // not needed for migration
|
||||
accessControl: accessControl,
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
|
||||
func NewDashboardSQLAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
namespacer request.NamespaceMapper,
|
||||
dashStore dashboards.Store,
|
||||
provisioning provisioning.ProvisioningService,
|
||||
@@ -90,7 +129,7 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc accesscontrol.DashboardPermissionsService,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
features featuremgmt.FeatureToggles,
|
||||
) DashboardAccess {
|
||||
) *dashboardSqlAccess {
|
||||
dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter)
|
||||
return &dashboardSqlAccess{
|
||||
sql: sql,
|
||||
@@ -101,7 +140,6 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider,
|
||||
dashboardPermissionSvc: dashboardPermissionSvc,
|
||||
libraryPanelSvc: libraryPanelSvc,
|
||||
accessControl: accessControl,
|
||||
log: log.New("dashboard.legacysql"),
|
||||
invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled),
|
||||
}
|
||||
}
|
||||
@@ -149,6 +187,290 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
|
||||
}, err
|
||||
}
|
||||
|
||||
// CountResources counts resources without migrating them
|
||||
func (a *dashboardSqlAccess) CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error) {
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ns, err := claims.ParseNamespace(opts.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orgId := ns.OrgID
|
||||
rsp := &resourcepb.BulkResponse{}
|
||||
err = sql.DB.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
|
||||
for _, res := range opts.Resources {
|
||||
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
|
||||
case "folder.grafana.app/folders":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = folders.GROUP
|
||||
summary.Group = folders.RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=TRUE AND org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/librarypanels":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.LIBRARY_PANEL_RESOURCE
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("library_element")+
|
||||
" WHERE org_id=?", orgId).Get(&summary.Count)
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
case "dashboard.grafana.app/dashboards":
|
||||
summary := &resourcepb.BulkResponse_Summary{}
|
||||
summary.Group = dashboardV1.GROUP
|
||||
summary.Resource = dashboardV1.DASHBOARD_RESOURCE
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
|
||||
_, err = sess.SQL("SELECT COUNT(*) FROM "+sql.Table("dashboard")+
|
||||
" WHERE is_folder=FALSE AND org_id=?", orgId).Get(&summary.Count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also count history
|
||||
_, err = sess.SQL(`SELECT COUNT(*)
|
||||
FROM `+sql.Table("dashboard_version")+` as dv
|
||||
JOIN `+sql.Table("dashboard")+` as dd
|
||||
ON dd.id = dv.dashboard_id
|
||||
WHERE org_id=?`, orgId).Get(&summary.History)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
// MigrateDashboards handles the dashboard migration logic
|
||||
func (a *dashboardSqlAccess) MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // include history
|
||||
AllowFallback: true, // allow fallback to dashboard table during migration
|
||||
Order: "ASC", // oldest first
|
||||
}
|
||||
|
||||
blobs := &BlobStoreInfo{}
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating dashboards...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
if dash.APIVersion == "" {
|
||||
dash.APIVersion = fmt.Sprintf("%s/v0alpha1", dashboardV1.GROUP)
|
||||
}
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error reading json from: %s // %w", rows.row.Dash.Name, err)
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.DASHBOARD_RESOURCE,
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%2d] %s (size:%d / %d|%d)", dash.Generation, dash.Name, len(req.Value), i, rows.count))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
opts.Progress(i, fmt.Sprintf("stream EOF/cancelled. index=%d", i))
|
||||
err = nil
|
||||
}
|
||||
return blobs, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(rows.rejected) > 0 {
|
||||
for _, row := range rows.rejected {
|
||||
id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID]
|
||||
a.log.Warn("rejected dashboard",
|
||||
"namespace", opts.Namespace,
|
||||
"dashboard", row.Dash.Name,
|
||||
"uid", row.Dash.UID,
|
||||
"id", id,
|
||||
"version", row.Dash.Generation,
|
||||
)
|
||||
opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name))
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return blobs, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished dashboards... (%d)", rows.count))
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// MigrateFolders handles the folder migration logic
|
||||
func (a *dashboardSqlAccess) MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
query := &DashboardQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 100000000,
|
||||
GetFolders: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
sql, err := a.sql(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opts.Progress(-1, "migrating folders...")
|
||||
rows, err := a.getRows(ctx, sql, query)
|
||||
if rows != nil {
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now send each dashboard
|
||||
for i := 1; rows.Next(); i++ {
|
||||
dash := rows.row.Dash
|
||||
dash.APIVersion = "folder.grafana.app/v1beta1"
|
||||
dash.Kind = "Folder"
|
||||
dash.SetNamespace(opts.Namespace)
|
||||
dash.SetResourceVersion("") // it will be filled in by the backend
|
||||
|
||||
spec := map[string]any{
|
||||
"title": dash.Spec.Object["title"],
|
||||
}
|
||||
description := dash.Spec.Object["description"]
|
||||
if description != nil {
|
||||
spec["description"] = description
|
||||
}
|
||||
dash.Spec.Object = spec
|
||||
|
||||
body, err := json.Marshal(dash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
Name: rows.Name(),
|
||||
},
|
||||
Value: body,
|
||||
Folder: rows.row.FolderUID,
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if dash.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
} else if dash.Generation < 0 {
|
||||
req.Action = resourcepb.BulkRequest_DELETED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", dash.Generation, dash.Name, len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Error() != nil {
|
||||
return nil, rows.Error()
|
||||
}
|
||||
|
||||
opts.Progress(-2, fmt.Sprintf("finished folders... (%d)", rows.count))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// MigrateLibraryPanels handles the library panel migration logic
|
||||
func (a *dashboardSqlAccess) MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error) {
|
||||
opts.Progress(-1, "migrating library panels...")
|
||||
panels, err := a.GetLibraryPanels(ctx, LibraryPanelQuery{
|
||||
OrgID: orgId,
|
||||
Limit: 1000000,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, panel := range panels.Items {
|
||||
meta, err := utils.MetaAccessor(&panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(panel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &resourcepb.BulkRequest{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Namespace: opts.Namespace,
|
||||
Group: dashboardV1.GROUP,
|
||||
Resource: dashboardV1.LIBRARY_PANEL_RESOURCE,
|
||||
Name: panel.Name,
|
||||
},
|
||||
Value: body,
|
||||
Folder: meta.GetFolder(),
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
}
|
||||
if panel.Generation > 1 {
|
||||
req.Action = resourcepb.BulkRequest_MODIFIED
|
||||
}
|
||||
|
||||
opts.Progress(i, fmt.Sprintf("[v:%d] %s (%d)", i, meta.GetName(), len(req.Value)))
|
||||
|
||||
err = stream.Send(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
opts.Progress(-2, fmt.Sprintf("finished panels... (%d)", len(panels.Items)))
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var _ resource.ListIterator = (*rowsWrapper)(nil)
|
||||
|
||||
type rowsWrapper struct {
|
||||
|
||||
@@ -21,6 +21,9 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/dashboards"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
)
|
||||
|
||||
func TestScanRow(t *testing.T) {
|
||||
@@ -529,3 +532,146 @@ func TestParseLibraryPanelRow(t *testing.T) {
|
||||
require.Nil(t, updatedTimestamp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDashboardMigrationQuery(t *testing.T) {
|
||||
// Test that migration queries use AllowFallback flag correctly
|
||||
nodb := &legacysql.LegacyDatabaseHelper{
|
||||
Table: func(n string) string {
|
||||
return "grafana." + n
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("Migration query should enable AllowFallback flag", func(t *testing.T) {
|
||||
// Create a migration query as would be used in actual migration
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true, // Migration includes history
|
||||
AllowFallback: true, // This is the key flag for migration
|
||||
Order: "ASC", // Migration uses ascending order
|
||||
}
|
||||
|
||||
// Verify UseHistoryTable returns true (requirement for COALESCE logic)
|
||||
require.True(t, migrationQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
|
||||
// Verify the flag is set correctly
|
||||
require.True(t, migrationQuery.AllowFallback, "Migration query should allow fallback")
|
||||
require.True(t, migrationQuery.GetHistory, "Migration query should get history")
|
||||
require.Equal(t, "ASC", migrationQuery.Order, "Migration should use ascending order")
|
||||
})
|
||||
|
||||
t.Run("Regular history query should not use AllowFallback", func(t *testing.T) {
|
||||
// Regular history query without migration
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
require.True(t, historyQuery.UseHistoryTable(), "History query should use history table")
|
||||
require.False(t, historyQuery.AllowFallback, "Regular history query should not allow fallback")
|
||||
require.True(t, historyQuery.GetHistory, "History query should get history")
|
||||
})
|
||||
|
||||
t.Run("Migration query template produces COALESCE SQL", func(t *testing.T) {
|
||||
// Test that the SQL template produces COALESCE logic for migration queries
|
||||
migrationQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
AllowFallback: true,
|
||||
Order: "ASC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, migrationQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
// Execute the template to get the generated SQL
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that COALESCE functions are present in the generated SQL
|
||||
// These should be used when GetHistory=true AND AllowFallback=true
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Migration SQL should contain COALESCE for updated timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version)",
|
||||
"Migration SQL should contain COALESCE for version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.data, dashboard.data)",
|
||||
"Migration SQL should contain COALESCE for data")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.api_version, dashboard.api_version)",
|
||||
"Migration SQL should contain COALESCE for api_version")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.message, '')",
|
||||
"Migration SQL should contain COALESCE for message with empty string fallback")
|
||||
|
||||
// Verify ORDER BY uses COALESCE as well
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED created timestamp")
|
||||
require.Contains(t, sql, "COALESCE(dashboard_version.version, dashboard.version) ASC",
|
||||
"Migration SQL should ORDER BY COALESCED version")
|
||||
|
||||
// Verify it doesn't have the strict history table filter that would exclude NULL version entries
|
||||
require.NotContains(t, sql, "dashboard_version.id IS NOT NULL",
|
||||
"Migration SQL should not exclude dashboards without version entries")
|
||||
})
|
||||
|
||||
t.Run("Regular history query produces strict SQL", func(t *testing.T) {
|
||||
// Test that regular history queries still use strict dashboard_version fields
|
||||
historyQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
GetHistory: true,
|
||||
Order: "DESC",
|
||||
}
|
||||
|
||||
req := newQueryReq(nodb, historyQuery)
|
||||
req.SQLTemplate = mocks.NewTestingSQLTemplate()
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(sqlQueryDashboards, &req)
|
||||
require.NoError(t, err)
|
||||
|
||||
sql := rawQuery
|
||||
|
||||
// Verify that direct dashboard_version fields are used (no COALESCE)
|
||||
require.Contains(t, sql, "dashboard_version.created as updated",
|
||||
"Regular history SQL should use direct dashboard_version.created")
|
||||
require.Contains(t, sql, "dashboard_version.version",
|
||||
"Regular history SQL should use direct dashboard_version.version")
|
||||
require.Contains(t, sql, "dashboard_version.data",
|
||||
"Regular history SQL should use direct dashboard_version.data")
|
||||
|
||||
// NOTE: We intentionally do NOT add dashboard_version.id IS NOT NULL filter
|
||||
// to allow for cases where dashboard_version entries might be missing
|
||||
|
||||
// Should not contain COALESCE functions
|
||||
require.NotContains(t, sql, "COALESCE(dashboard_version.created, dashboard.updated)",
|
||||
"Regular history SQL should not contain COALESCE for updated")
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateDashboardsConfiguration(t *testing.T) {
|
||||
// Test the actual migration function configuration
|
||||
|
||||
t.Run("Migration options should configure query correctly", func(t *testing.T) {
|
||||
// Test the migration configuration as used in real migration
|
||||
opts := MigrateOptions{
|
||||
WithHistory: true, // Migration includes history
|
||||
}
|
||||
|
||||
// This simulates what happens in migrateDashboards function
|
||||
expectedQuery := &DashboardQuery{
|
||||
OrgID: 1,
|
||||
Limit: 100000000,
|
||||
GetHistory: opts.WithHistory, // Should be true
|
||||
AllowFallback: true, // Should be true for migration
|
||||
Order: "ASC", // Should be ASC for migration
|
||||
}
|
||||
|
||||
// Verify the configuration matches what migration sets up
|
||||
require.True(t, expectedQuery.GetHistory, "Migration should enable GetHistory")
|
||||
require.True(t, expectedQuery.AllowFallback, "Migration should enable AllowFallback")
|
||||
require.Equal(t, "ASC", expectedQuery.Order, "Migration should use ascending order")
|
||||
require.Equal(t, 100000000, expectedQuery.Limit, "Migration should use large limit")
|
||||
|
||||
// Verify UseHistoryTable logic
|
||||
require.True(t, expectedQuery.UseHistoryTable(), "Migration query should use history table")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -55,10 +55,9 @@ type LibraryPanelQuery struct {
|
||||
LastID int64
|
||||
}
|
||||
|
||||
type DashboardAccess interface {
|
||||
type DashboardAccessor interface {
|
||||
resource.StorageBackend
|
||||
resourcepb.ResourceIndexServer
|
||||
LegacyMigrator
|
||||
|
||||
GetDashboard(ctx context.Context, orgId int64, uid string, version int64) (*dashboardV1.Dashboard, int64, error)
|
||||
SaveDashboard(ctx context.Context, orgId int64, dash *dashboardV1.Dashboard, failOnExisting bool) (*dashboardV1.Dashboard, bool, error)
|
||||
@@ -67,3 +66,12 @@ type DashboardAccess interface {
|
||||
// Get a typed list
|
||||
GetLibraryPanels(ctx context.Context, query LibraryPanelQuery) (*dashboardV0.LibraryPanelList, error)
|
||||
}
|
||||
|
||||
//go:generate mockery --name MigrationDashboardAccessor --structname MockMigrationDashboardAccessor --inpackage --filename migration_dashboard_accessor_mock.go --with-expecter
|
||||
type MigrationDashboardAccessor interface {
|
||||
// Migration helper methods - these support the separate LegacyMigrator
|
||||
CountResources(ctx context.Context, opts MigrateOptions) (*resourcepb.BulkResponse, error)
|
||||
MigrateDashboards(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateFolders(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
MigrateLibraryPanels(ctx context.Context, orgId int64, opts MigrateOptions, stream resourcepb.BulkStore_BulkProcessClient) (*BlobStoreInfo, error)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
type DashboardStorage struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
DashboardService dashboards.DashboardService
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ var (
|
||||
)
|
||||
|
||||
type LibraryPanelStore struct {
|
||||
Access legacy.DashboardAccess
|
||||
Access legacy.DashboardAccessor
|
||||
ResourceInfo utils.ResourceInfo
|
||||
service libraryelements.Service
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ func RegisterAPIService(
|
||||
publicDashboardService: publicDashboardService,
|
||||
|
||||
legacy: &DashboardStorage{
|
||||
Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
Access: legacy.NewDashboardSQLAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, dashboardPermissionsSvc, accessControl, features),
|
||||
DashboardService: dashboardService,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ type dtoBuilder = func(dashboard runtime.Object, access *dashboard.DashboardAcce
|
||||
// The DTO returns everything the UI needs in a single request
|
||||
type DTOConnector struct {
|
||||
getter rest.Getter
|
||||
legacy legacy.DashboardAccess
|
||||
legacy legacy.DashboardAccessor
|
||||
unified resource.ResourceClient
|
||||
largeObjects apistore.LargeObjectSupport
|
||||
accessControl accesscontrol.AccessControl
|
||||
@@ -42,7 +42,7 @@ type DTOConnector struct {
|
||||
func NewDTOConnector(
|
||||
getter rest.Getter,
|
||||
largeObjects apistore.LargeObjectSupport,
|
||||
legacyAccess legacy.DashboardAccess,
|
||||
legacyAccess legacy.DashboardAccessor,
|
||||
resourceClient resource.ResourceClient,
|
||||
accessControl accesscontrol.AccessControl,
|
||||
scheme *runtime.Scheme,
|
||||
|
||||
Reference in New Issue
Block a user