public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CloudMigrations: Refactor folder name resolution when fetching migration json data (#102604)

Browse Source 
* CloudMigrations: Refactor folder name resolution when fetching migration json data

* Update pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go

---------

Co-authored-by: Michael Mandrus <41969079+mmandrus@users.noreply.github.com>
This commit is contained in:
Matheus Macabu
2025-03-24 08:17:11 +01:00
committed by GitHub
co-authored by Michael Mandrus
parent 543c0bbccb
commit 9c0c9359a0
2 changed files with 169 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
+65 -35
View File
@@ -628,62 +628,92 @@ func TestGetFolderNamesForFolderUIDs(t *testing.T) {
func TestGetParentNames(t *testing.T) {
t.Parallel()
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
s := setUpServiceTest(t, false).(*Service)
user := &user.SignedInUser{OrgID: 1}
libraryElementFolderUID := "folderUID-A"
testcases := []struct {
name string
fakeFolders []*folder.Folder
folders []folder.CreateFolderCommand
dashboards []dashboards.Dashboard
libraryElements []libraryElement
alertRules []alertRule
expectedParentNames map[cloudmigration.MigrateDataType][]string
folderHierarchy map[cloudmigration.MigrateDataType]map[string]string
expectedParentNames map[cloudmigration.MigrateDataType]map[string]string
}{
{
name: "multiple data types",
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1, ParentUID: ""},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1, ParentUID: "folderUID-A"},
{UID: "folderUID-X", Title: "Folder X", OrgID: 1, ParentUID: ""},
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1},
{UID: "folderUID-C", Title: "Folder C", OrgID: 1},
},
folders: []folder.CreateFolderCommand{
{UID: "folderUID-C", Title: "Folder A", OrgID: 1, ParentUID: "folderUID-A"},
folderHierarchy: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "folderUID-A", "dashboard-2": "folderUID-B", "dashboard-3": ""},
cloudmigration.LibraryElementDataType: {"libElement-1": "folderUID-A", "libElement-2": "folderUID-C"},
cloudmigration.AlertRuleType: {"alertRule-1": "folderUID-B"},
},
dashboards: []dashboards.Dashboard{
{UID: "dashboardUID-0", OrgID: 1, FolderUID: ""},
{UID: "dashboardUID-1", OrgID: 1, FolderUID: "folderUID-A"},
{UID: "dashboardUID-2", OrgID: 1, FolderUID: "folderUID-B"},
expectedParentNames: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "Folder A", "dashboard-2": "Folder B", "dashboard-3": ""},
cloudmigration.LibraryElementDataType: {"libElement-1": "Folder A", "libElement-2": "Folder C"},
cloudmigration.AlertRuleType: {"alertRule-1": "Folder B"},
},
libraryElements: []libraryElement{
{UID: "libraryElementUID-0", FolderUID: &libraryElementFolderUID},
{UID: "libraryElementUID-1"},
},
{
name: "empty folder hierarchy",
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
alertRules: []alertRule{
{UID: "alertRuleUID-0", FolderUID: ""},
{UID: "alertRuleUID-1", FolderUID: "folderUID-B"},
folderHierarchy: map[cloudmigration.MigrateDataType]map[string]string{},
expectedParentNames: map[cloudmigration.MigrateDataType]map[string]string{},
},
{
name: "all root folders (no parents)",
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
expectedParentNames: map[cloudmigration.MigrateDataType][]string{
cloudmigration.DashboardDataType: {"", "Folder A", "Folder B"},
cloudmigration.FolderDataType: {"Folder A"},
cloudmigration.LibraryElementDataType: {"Folder A"},
cloudmigration.AlertRuleType: {"Folder B"},
folderHierarchy: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "", "dashboard-2": ""},
cloudmigration.LibraryElementDataType: {"libElement-1": ""},
},
expectedParentNames: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "", "dashboard-2": ""},
cloudmigration.LibraryElementDataType: {"libElement-1": ""},
},
},
{
name: "non-existent folder UIDs",
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
folderHierarchy: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "folderUID-A", "dashboard-2": "non-existent-uid"},
},
expectedParentNames: map[cloudmigration.MigrateDataType]map[string]string{
cloudmigration.DashboardDataType: {"dashboard-1": "Folder A", "dashboard-2": ""},
},
},
}
for _, tc := range testcases {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.fakeFolders}
t.Run(tc.name, func(t *testing.T) {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.fakeFolders}
dataUIDsToParentNamesByType, err := s.getParentNames(ctx, user, tc.dashboards, tc.folders, tc.libraryElements, tc.alertRules)
require.NoError(t, err)
dataUIDsToParentNamesByType, err := s.getParentNames(ctx, user, tc.folderHierarchy)
require.NoError(t, err)
for dataType, expectedParentNames := range tc.expectedParentNames {
actualParentNames := slices.Collect(maps.Values(dataUIDsToParentNamesByType[dataType]))
require.Len(t, actualParentNames, len(expectedParentNames))
require.ElementsMatch(t, expectedParentNames, actualParentNames)
}
for dataType, expectedParentNames := range tc.expectedParentNames {
actualParentNames := dataUIDsToParentNamesByType[dataType]
require.Equal(t, len(expectedParentNames), len(actualParentNames))
for uid, expectedName := range expectedParentNames {
actualName, exists := actualParentNames[uid]
require.True(t, exists)
require.Equal(t, expectedName, actualName)
}
}
})
}
}
pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go
+104 -108
View File
@@ -12,6 +12,9 @@ import (
"sort"
"time"
"go.opentelemetry.io/otel/codes"
"golang.org/x/crypto/nacl/box"
snapshot "github.com/grafana/grafana-cloud-migration-snapshot/src"
"github.com/grafana/grafana-cloud-migration-snapshot/src/contracts"
"github.com/grafana/grafana-cloud-migration-snapshot/src/infra/crypto"
@@ -29,9 +32,6 @@ import (
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/util/retryer"
"golang.org/x/crypto/nacl/box"
"go.opentelemetry.io/otel/codes"
)
var currentMigrationTypes = []cloudmigration.MigrateDataType{
@@ -52,6 +52,10 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.getMigrationDataJSON")
defer span.End()
migrationDataSlice := make([]cloudmigration.MigrateDataRequestItem, 0)
folderHierarchy := make(map[cloudmigration.MigrateDataType]map[string]string, 0)
// Plugins
plugins, err := s.getPlugins(ctx, signedInUser)
if err != nil {
@@ -59,74 +63,6 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
return nil, err
}
// Data sources
dataSources, err := s.getDataSourceCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get datasources", "err", err)
return nil, err
}
// Dashboards and folders are linked via the schema, so we need to get both
dashs, folders, err := s.getDashboardAndFolderCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get dashboards and folders", "err", err)
return nil, err
}
libraryElements, err := s.getLibraryElementsCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get library elements", "err", err)
return nil, err
}
// Alerts: Mute Timings
muteTimings, err := s.getAlertMuteTimings(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert mute timings", "err", err)
return nil, err
}
// Alerts: Notification Templates
notificationTemplates, err := s.getNotificationTemplates(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert notification templates", "err", err)
return nil, err
}
// Alerts: Contact Points
contactPoints, err := s.getContactPoints(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert contact points", "err", err)
return nil, err
}
// Alerts: Notification Policies
notificationPolicies, err := s.getNotificationPolicies(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert notification policies", "err", err)
return nil, err
}
// Alerts: Alert Rule Groups
alertRuleGroups, err := s.getAlertRuleGroups(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert rule groups", "err", err)
return nil, err
}
// Alerts: Alert Rules
alertRules, err := s.getAlertRules(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert rules", "err", err)
return nil, err
}
migrationDataSlice := make(
[]cloudmigration.MigrateDataRequestItem, 0,
len(plugins)+len(dataSources)+len(dashs)+len(folders)+len(libraryElements)+
len(muteTimings)+len(notificationTemplates)+len(contactPoints)+len(alertRules),
)
for _, plugin := range plugins {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.PluginDataType,
@@ -136,6 +72,13 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Data sources
dataSources, err := s.getDataSourceCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get datasources", "err", err)
return nil, err
}
for _, ds := range dataSources {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DatasourceDataType,
@@ -145,6 +88,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Dashboards & Folders: linked via the schema, so we need to get both
dashs, folders, err := s.getDashboardAndFolderCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get dashboards and folders", "err", err)
return nil, err
}
folderHierarchy[cloudmigration.DashboardDataType] = make(map[string]string, 0)
for _, dashboard := range dashs {
dashboard.Data.Del("id")
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
@@ -159,8 +111,12 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
FolderUID: dashboard.FolderUID,
},
})
folderHierarchy[cloudmigration.DashboardDataType][dashboard.UID] = dashboard.FolderUID
}
folderHierarchy[cloudmigration.FolderDataType] = make(map[string]string, 0)
folders = sortFolders(folders)
for _, f := range folders {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
@@ -169,8 +125,19 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
Name: f.Title,
Data: f,
})
folderHierarchy[cloudmigration.FolderDataType][f.UID] = f.ParentUID
}
// Library Elements
libraryElements, err := s.getLibraryElementsCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get library elements", "err", err)
return nil, err
}
folderHierarchy[cloudmigration.LibraryElementDataType] = make(map[string]string, 0)
for _, libraryElement := range libraryElements {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.LibraryElementDataType,
@@ -178,6 +145,17 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
Name: libraryElement.Name,
Data: libraryElement,
})
if libraryElement.FolderUID != nil {
folderHierarchy[cloudmigration.LibraryElementDataType][libraryElement.UID] = *libraryElement.FolderUID
}
}
// Alerts: Mute Timings
muteTimings, err := s.getAlertMuteTimings(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert mute timings", "err", err)
return nil, err
}
for _, muteTiming := range muteTimings {
@@ -189,6 +167,13 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Alerts: Notification Templates
notificationTemplates, err := s.getNotificationTemplates(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert notification templates", "err", err)
return nil, err
}
for _, notificationTemplate := range notificationTemplates {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.NotificationTemplateType,
@@ -198,6 +183,13 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Alerts: Contact Points
contactPoints, err := s.getContactPoints(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert contact points", "err", err)
return nil, err
}
for _, contactPoint := range contactPoints {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.ContactPointType,
@@ -207,6 +199,13 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Alerts: Notification Policies
notificationPolicies, err := s.getNotificationPolicies(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert notification policies", "err", err)
return nil, err
}
if len(notificationPolicies.Name) > 0 {
// Notification Policy can only be managed by updating its entire tree, so we send the whole thing as one item.
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
@@ -217,6 +216,13 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Alerts: Alert Rule Groups
alertRuleGroups, err := s.getAlertRuleGroups(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert rule groups", "err", err)
return nil, err
}
for _, alertRuleGroup := range alertRuleGroups {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.AlertRuleGroupType,
@@ -226,6 +232,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Alerts: Alert Rules
alertRules, err := s.getAlertRules(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get alert rules", "err", err)
return nil, err
}
folderHierarchy[cloudmigration.AlertRuleType] = make(map[string]string, 0)
for _, alertRule := range alertRules {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.AlertRuleType,
@@ -233,10 +248,12 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
Name: alertRule.Title,
Data: alertRule,
})
folderHierarchy[cloudmigration.AlertRuleType][alertRule.UID] = alertRule.FolderUID
}
// Obtain the names of parent elements for Dashboard and Folders data types
parentNamesByType, err := s.getParentNames(ctx, signedInUser, dashs, folders, libraryElements, alertRules)
// Obtain the names of parent elements for data types that have folders.
parentNamesByType, err := s.getParentNames(ctx, signedInUser, folderHierarchy)
if err != nil {
s.log.Error("Failed to get parent folder names", "err", err)
}
@@ -805,10 +822,7 @@ func (s *Service) getFolderNamesForFolderUIDs(ctx context.Context, signedInUser
func (s *Service) getParentNames(
ctx context.Context,
signedInUser *user.SignedInUser,
dashboards []dashboards.Dashboard,
folders []folder.CreateFolderCommand,
libraryElements []libraryElement,
alertRules []alertRule,
folderHierarchy map[cloudmigration.MigrateDataType]map[string]string,
) (map[cloudmigration.MigrateDataType]map[string](string), error) {
parentNamesByType := make(map[cloudmigration.MigrateDataType]map[string]string)
for _, dataType := range currentMigrationTypes {
@@ -817,25 +831,18 @@ func (s *Service) getParentNames(
// Obtain list of unique folderUIDs
parentFolderUIDsSet := make(map[string]struct{})
for _, dashboard := range dashboards {
// we dont need the root folder
if dashboard.FolderUID != "" {
parentFolderUIDsSet[dashboard.FolderUID] = struct{}{}
}
}
for _, f := range folders {
parentFolderUIDsSet[f.ParentUID] = struct{}{}
}
for _, libraryElement := range libraryElements {
if libraryElement.FolderUID != nil {
parentFolderUIDsSet[*libraryElement.FolderUID] = struct{}{}
}
}
for _, alertRule := range alertRules {
if alertRule.FolderUID != "" {
parentFolderUIDsSet[alertRule.FolderUID] = struct{}{}
for _, folderUIDs := range folderHierarchy {
for _, folderUID := range folderUIDs {
// Skip the root folder
if folderUID == "" {
continue
}
parentFolderUIDsSet[folderUID] = struct{}{}
}
}
parentFolderUIDsSlice := make([]string, 0, len(parentFolderUIDsSet))
for parentFolderUID := range parentFolderUIDsSet {
parentFolderUIDsSlice = append(parentFolderUIDsSlice, parentFolderUID)
@@ -849,20 +856,9 @@ func (s *Service) getParentNames(
}
// Prepare map of {data type: {data UID : parentName}}
for _, dashboard := range dashboards {
parentNamesByType[cloudmigration.DashboardDataType][dashboard.UID] = foldersUIDsToFolderName[dashboard.FolderUID]
}
for _, f := range folders {
parentNamesByType[cloudmigration.FolderDataType][f.UID] = foldersUIDsToFolderName[f.ParentUID]
}
for _, libraryElement := range libraryElements {
if libraryElement.FolderUID != nil {
parentNamesByType[cloudmigration.LibraryElementDataType][libraryElement.UID] = foldersUIDsToFolderName[*libraryElement.FolderUID]
}
}
for _, alertRule := range alertRules {
if alertRule.FolderUID != "" {
parentNamesByType[cloudmigration.AlertRuleType][alertRule.UID] = foldersUIDsToFolderName[alertRule.FolderUID]
for dataType, uidFolderMap := range folderHierarchy {
for uid, folderUID := range uidFolderMap {
parentNamesByType[dataType][uid] = foldersUIDsToFolderName[folderUID]
}
}