public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add unified storage data migration step for playlists (#114582)

Browse Source 
* fix: add type

* feat: register step

* feat: add playlist support

* test: add test case

* fix: gen mock

* fix: go gen

* fix: lint

* fix: lint

* fix: tests

* fix: add resource

* fix: readd

* fix: address comments

* fix: independent playlist query for migrations

* fix: remove lock logic for sqlite

* fix: handle creation and update datetimes

* fix: query templating

* fix: simply resources and address comments
This commit is contained in:
Mustafa Sencer Özcan
2025-12-04 15:15:00 +00:00
committed by GitHub
parent e5259c2ad4
commit 4c5d9cb95f
16 changed files with 641 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/migrations/migrator.go
+18 -64
View File
@@ -10,8 +10,6 @@ import (
authlib "github.com/grafana/authlib/types"
v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
@@ -35,39 +33,26 @@ type streamProvider interface {
createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error)
}
// resourceClientStreamProvider creates streams using resource.ResourceClient
type resourceClientStreamProvider struct {
client resource.ResourceClient
}
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
// Build collection settings for resource client
func buildCollectionSettings(opts legacy.MigrateOptions) resource.BulkSettings {
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
Resource: folders.RESOURCE,
})
case "dashboard.grafana.app/librarypanels":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
})
case "dashboard.grafana.app/dashboards":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.DASHBOARD_RESOURCE,
})
key := buildResourceKey(res.Group, res.Resource, opts.Namespace)
if key != nil {
settings.Collection = append(settings.Collection, key)
}
}
return settings
}
type resourceClientStreamProvider struct {
client resource.ResourceClient
}
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
settings := buildCollectionSettings(opts)
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
return r.client.BulkProcess(ctx)
}
@@ -78,33 +63,7 @@ type bulkStoreClientStreamProvider struct {
}
func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
// Build collection settings for resource client
settings := resource.BulkSettings{
RebuildCollection: true,
SkipValidation: true,
}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: folders.GROUP,
Resource: folders.RESOURCE,
})
case "dashboard.grafana.app/librarypanels":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.LIBRARY_PANEL_RESOURCE,
})
case "dashboard.grafana.app/dashboards":
settings.Collection = append(settings.Collection, &resourcepb.ResourceKey{
Namespace: opts.Namespace,
Group: v1beta1.GROUP,
Resource: v1beta1.DASHBOARD_RESOURCE,
})
}
}
settings := buildCollectionSettings(opts)
ctx = metadata.NewOutgoingContext(ctx, settings.ToMD())
return b.client.BulkProcess(ctx)
}
@@ -170,16 +129,11 @@ func (m *unifiedMigration) Migrate(ctx context.Context, opts legacy.MigrateOptio
migratorFuncs := []migratorFunc{}
for _, res := range opts.Resources {
switch fmt.Sprintf("%s/%s", res.Group, res.Resource) {
case "folder.grafana.app/folders":
migratorFuncs = append(migratorFuncs, m.MigrateFolders)
case "dashboard.grafana.app/librarypanels":
migratorFuncs = append(migratorFuncs, m.MigrateLibraryPanels)
case "dashboard.grafana.app/dashboards":
migratorFuncs = append(migratorFuncs, m.MigrateDashboards)
default:
return nil, fmt.Errorf("unsupported resource: %s", res)
fn := getMigratorFunc(m.MigrationDashboardAccessor, res.Group, res.Resource)
if fn == nil {
return nil, fmt.Errorf("unsupported resource: %s/%s", res.Group, res.Resource)
}
migratorFuncs = append(migratorFuncs, fn)
}
// Execute migrations
pkg/storage/unified/migrations/migrator_test.go
+1
View File
@@ -45,6 +45,7 @@ func TestIntegrationMigrations(t *testing.T) {
migrationTestCases := []resourceMigratorTestCase{
newFoldersAndDashboardsTestCase(),
newPlaylistsTestCase(),
}
runMigrationTestSuite(t, migrationTestCases)
pkg/storage/unified/migrations/playlists_test.go
+119
View File
@@ -0,0 +1,119 @@
package migrations_test
import (
"context"
"testing"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/playlist"
"github.com/grafana/grafana/pkg/services/playlist/playlistimpl"
"github.com/grafana/grafana/pkg/tests/apis"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// playlistsTestCase tests the "playlists" ResourceMigration
type playlistsTestCase struct {
playlistUIDs []string
}
// newPlaylistsTestCase creates a test case for the playlists migrator
func newPlaylistsTestCase() resourceMigratorTestCase {
return &playlistsTestCase{
playlistUIDs: []string{},
}
}
func (tc *playlistsTestCase) name() string {
return "playlists"
}
func (tc *playlistsTestCase) resources() []schema.GroupVersionResource {
return []schema.GroupVersionResource{
{
Group: "playlist.grafana.app",
Version: "v0alpha1",
Resource: "playlists",
},
}
}
func (tc *playlistsTestCase) setup(t *testing.T, helper *apis.K8sTestHelper) {
t.Helper()
// Get playlist service from the test environment
// The service writes directly to SQL storage, which works in Mode0
env := helper.GetEnv()
playlistSvc := playlistimpl.ProvideService(env.SQLStore, tracing.InitializeTracerForTest())
// Use a non-existent dashboard UID for testing
// This avoids interfering with other test cases
nonExistentDashboardUID := "non-existent-dashboard-uid"
// Create playlist with dashboard UID items (pointing to non-existent dashboard)
playlist1UID := createTestPlaylist(t, playlistSvc, helper.Org1.OrgID, "Playlist with Dashboard UIDs", "5m", []playlist.PlaylistItem{
{Type: "dashboard_by_uid", Value: nonExistentDashboardUID, Order: 1},
})
tc.playlistUIDs = append(tc.playlistUIDs, playlist1UID)
// Create playlist with tag items
playlist2UID := createTestPlaylist(t, playlistSvc, helper.Org1.OrgID, "Playlist with Tags", "10m", []playlist.PlaylistItem{
{Type: "dashboard_by_tag", Value: "test-tag", Order: 1},
{Type: "dashboard_by_tag", Value: "another-tag", Order: 2},
})
tc.playlistUIDs = append(tc.playlistUIDs, playlist2UID)
// Create playlist with mixed items
playlist3UID := createTestPlaylist(t, playlistSvc, helper.Org1.OrgID, "Playlist with Mixed Items", "15m", []playlist.PlaylistItem{
{Type: "dashboard_by_uid", Value: nonExistentDashboardUID, Order: 1},
{Type: "dashboard_by_tag", Value: "mixed-tag", Order: 2},
})
tc.playlistUIDs = append(tc.playlistUIDs, playlist3UID)
}
func (tc *playlistsTestCase) verify(t *testing.T, helper *apis.K8sTestHelper, shouldExist bool) {
t.Helper()
expectedPlaylistCount := 0
if shouldExist {
expectedPlaylistCount = len(tc.playlistUIDs)
}
orgID := helper.Org1.OrgID
namespace := authlib.OrgNamespaceFormatter(orgID)
// Verify playlists
playlistCli := helper.GetResourceClient(apis.ResourceClientArgs{
User: helper.Org1.Admin,
Namespace: namespace,
GVR: schema.GroupVersionResource{
Group: "playlist.grafana.app",
Version: "v0alpha1",
Resource: "playlists",
},
})
verifyResourceCount(t, playlistCli, expectedPlaylistCount)
for _, uid := range tc.playlistUIDs {
verifyResource(t, playlistCli, uid, shouldExist)
}
}
func createTestPlaylist(t *testing.T, playlistSvc playlist.Service, orgID int64, name, interval string, items []playlist.PlaylistItem) string {
t.Helper()
cmd := &playlist.CreatePlaylistCommand{
Name: name,
Interval: interval,
Items: items,
OrgId: orgID,
}
result, err := playlistSvc.Create(context.Background(), cmd)
require.NoError(t, err)
require.NotNil(t, result)
require.NotEmpty(t, result.UID)
return result.UID
}
pkg/storage/unified/migrations/resources.go
+100
View File
@@ -0,0 +1,100 @@
package migrations
import (
"fmt"
v1beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
playlists "github.com/grafana/grafana/apps/playlist/pkg/apis/playlist/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"k8s.io/apimachinery/pkg/runtime/schema"
)
type ResourceDefinition struct {
GroupResource schema.GroupResource
MigratorFunc string // Name of the method: "MigrateFolders", "MigrateDashboards", etc.
}
var registeredResources = []ResourceDefinition{
{
GroupResource: schema.GroupResource{Group: folders.GROUP, Resource: folders.RESOURCE},
MigratorFunc: "MigrateFolders",
},
{
GroupResource: schema.GroupResource{Group: v1beta1.GROUP, Resource: v1beta1.LIBRARY_PANEL_RESOURCE},
MigratorFunc: "MigrateLibraryPanels",
},
{
GroupResource: schema.GroupResource{Group: v1beta1.GROUP, Resource: v1beta1.DASHBOARD_RESOURCE},
MigratorFunc: "MigrateDashboards",
},
{
GroupResource: schema.GroupResource{Group: playlists.APIGroup, Resource: "playlists"},
MigratorFunc: "MigratePlaylists",
},
}
func getResourceDefinition(group, resource string) *ResourceDefinition {
for i := range registeredResources {
r := &registeredResources[i]
if r.GroupResource.Group == group && r.GroupResource.Resource == resource {
return r
}
}
return nil
}
func buildResourceKey(group, resource, namespace string) *resourcepb.ResourceKey {
def := getResourceDefinition(group, resource)
if def == nil {
return nil
}
return &resourcepb.ResourceKey{
Namespace: namespace,
Group: def.GroupResource.Group,
Resource: def.GroupResource.Resource,
}
}
func getMigratorFunc(accessor legacy.MigrationDashboardAccessor, group, resource string) migratorFunc {
def := getResourceDefinition(group, resource)
if def == nil {
return nil
}
switch def.MigratorFunc {
case "MigrateFolders":
return accessor.MigrateFolders
case "MigrateLibraryPanels":
return accessor.MigrateLibraryPanels
case "MigrateDashboards":
return accessor.MigrateDashboards
case "MigratePlaylists":
return accessor.MigratePlaylists
default:
return nil
}
}
func validateRegisteredResources() error {
registeredMap := make(map[string]bool)
for _, gr := range registeredResources {
key := fmt.Sprintf("%s.%s", gr.GroupResource.Resource, gr.GroupResource.Group)
registeredMap[key] = true
}
var missing []string
for _, expected := range setting.MigratedUnifiedResources {
if !registeredMap[expected] {
missing = append(missing, expected)
}
}
if len(missing) > 0 {
return fmt.Errorf("resources declared in setting.MigratedUnifiedResources are not registered for migration: %v", missing)
}
return nil
}
pkg/storage/unified/migrations/service.go
+33 -12
View File
@@ -85,18 +85,18 @@ func RegisterMigrations(
logger.Warn("Failed to register migrator metrics", "error", err)
}
if err := validateRegisteredResources(); err != nil {
return err
}
// Register resource migrations
registerDashboardAndFolderMigration(mg, migrator, client)
registerPlaylistMigration(mg, migrator, client)
// Run all registered migrations (blocking)
sec := cfg.Raw.Section("database")
migrationLocking := sec.Key("migration_locking").MustBool(true)
if mg.Dialect.DriverName() == sqlstoremigrator.SQLite {
// disable migration locking for SQLite to avoid "database is locked" errors in the bulk operations
migrationLocking = false
}
if err := mg.RunMigrations(ctx,
migrationLocking,
sec.Key("migration_locking").MustBool(true),
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
return fmt.Errorf("unified storage data migration failed: %w", err)
}
@@ -106,13 +106,13 @@ func RegisterMigrations(
}
func registerDashboardAndFolderMigration(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) {
folders := schema.GroupResource{Group: "folder.grafana.app", Resource: "folders"}
dashboards := schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}
foldersDef := getResourceDefinition("folder.grafana.app", "folders")
dashboardsDef := getResourceDefinition("dashboard.grafana.app", "dashboards")
driverName := mg.Dialect.DriverName()
folderCountValidator := NewCountValidator(
client,
folders,
foldersDef.GroupResource,
"dashboard",
"org_id = ? and is_folder = true",
driverName,
@@ -120,19 +120,40 @@ func registerDashboardAndFolderMigration(mg *sqlstoremigrator.Migrator, migrator
dashboardCountValidator := NewCountValidator(
client,
dashboards,
dashboardsDef.GroupResource,
"dashboard",
"org_id = ? and is_folder = false",
driverName,
)
folderTreeValidator := NewFolderTreeValidator(client, folders, driverName)
folderTreeValidator := NewFolderTreeValidator(client, foldersDef.GroupResource, driverName)
dashboardsAndFolders := NewResourceMigration(
migrator,
[]schema.GroupResource{folders, dashboards},
[]schema.GroupResource{foldersDef.GroupResource, dashboardsDef.GroupResource},
"folders-dashboards",
[]Validator{folderCountValidator, dashboardCountValidator, folderTreeValidator},
)
mg.AddMigration("folders and dashboards migration", dashboardsAndFolders)
}
func registerPlaylistMigration(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) {
playlistsDef := getResourceDefinition("playlist.grafana.app", "playlists")
driverName := mg.Dialect.DriverName()
playlistCountValidator := NewCountValidator(
client,
playlistsDef.GroupResource,
"playlist",
"org_id = ?",
driverName,
)
playlistsMigration := NewResourceMigration(
migrator,
[]schema.GroupResource{playlistsDef.GroupResource},
"playlists",
[]Validator{playlistCountValidator},
)
mg.AddMigration("playlists migration", playlistsMigration)
}