public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CloudMigrations: Store parent folder name in cloud_migration_resource table (#94009)

Browse Source 
* use name in fe

* store parent folder name in local db

* clean up

* tiny test

* trial react

* rename to parent name

* go lint

* generate api and ts

* go tests

* rearrange

* clean

* update with suggestions from josh

* make library elements work

* updates from comments

* global migration types

* parent name for alter table
This commit is contained in:
Dana Axinte
2024-10-07 11:35:08 +01:00
committed by GitHub
parent 5a9bd1d1cf
commit d88be2819d
16 changed files with 309 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
+121
View File
@@ -2,8 +2,10 @@ package cloudmigrationimpl
import (
"context"
"maps"
"os"
"path/filepath"
"slices"
"testing"
"time"
@@ -392,6 +394,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
Results: []cloudmigration.CloudMigrationResource{
{
Name: "1 name",
ParentName: "1 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "1", // this will be core
Status: cloudmigration.ItemStatusOK,
@@ -399,6 +402,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "2 name",
ParentName: "",
Type: cloudmigration.DatasourceDataType,
RefID: "2", // this will be non-core
Status: cloudmigration.ItemStatusOK,
@@ -406,6 +410,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "3 name",
ParentName: "3 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "3", // this will be non-core with an error
Status: cloudmigration.ItemStatusError,
@@ -414,6 +419,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "4 name",
ParentName: "4 folder name",
Type: cloudmigration.DatasourceDataType,
RefID: "4", // this will be deleted
Status: cloudmigration.ItemStatusOK,
@@ -564,6 +570,121 @@ func TestReportEvent(t *testing.T) {
require.Equal(t, 1, gmsMock.reportEventCalled)
})
}
func TestGetFolderNamesForFolderUIDs(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
user := &user.SignedInUser{OrgID: 1}
testcases := []struct {
folders []*folder.Folder
folderUIDs []string
expectedFolderNames []string
}{
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1},
},
folderUIDs: []string{"folderUID-A", "folderUID-B"},
expectedFolderNames: []string{"Folder A", "Folder B"},
},
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
folderUIDs: []string{"folderUID-A"},
expectedFolderNames: []string{"Folder A"},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{"folderUID-A"},
expectedFolderNames: []string{""},
},
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
folderUIDs: []string{"folderUID-A", "folderUID-B"},
expectedFolderNames: []string{"Folder A", ""},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{""},
expectedFolderNames: []string{""},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{},
expectedFolderNames: []string{},
},
}
for _, tc := range testcases {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.folders}
folderUIDsToFolders, err := s.getFolderNamesForFolderUIDs(ctx, user, tc.folderUIDs)
require.NoError(t, err)
resFolderNames := slices.Collect(maps.Values(folderUIDsToFolders))
require.Len(t, resFolderNames, len(tc.expectedFolderNames))
require.ElementsMatch(t, resFolderNames, tc.expectedFolderNames)
}
}
func TestGetParentNames(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
user := &user.SignedInUser{OrgID: 1}
libraryElementFolderUID := "folderUID-A"
testcases := []struct {
fakeFolders []*folder.Folder
folders []folder.CreateFolderCommand
dashboards []dashboards.Dashboard
libraryElements []libraryElement
expectedDashParentNames []string
expectedFoldParentNames []string
}{
{
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1, ParentUID: ""},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1, ParentUID: "folderUID-A"},
},
folders: []folder.CreateFolderCommand{
{UID: "folderUID-C", Title: "Folder A", OrgID: 1, ParentUID: "folderUID-A"},
},
dashboards: []dashboards.Dashboard{
{UID: "dashboardUID-0", OrgID: 1, FolderUID: ""},
{UID: "dashboardUID-1", OrgID: 1, FolderUID: "folderUID-A"},
{UID: "dashboardUID-2", OrgID: 1, FolderUID: "folderUID-B"},
},
libraryElements: []libraryElement{
{UID: "libraryElementUID-0", FolderUID: &libraryElementFolderUID},
},
expectedDashParentNames: []string{"", "Folder A", "Folder B"},
expectedFoldParentNames: []string{"Folder A"},
},
}
for _, tc := range testcases {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.fakeFolders}
dataUIDsToParentNamesByType, err := s.getParentNames(ctx, user, tc.dashboards, tc.folders, tc.libraryElements)
require.NoError(t, err)
resDashParentNames := slices.Collect(maps.Values(dataUIDsToParentNamesByType[cloudmigration.DashboardDataType]))
require.Len(t, resDashParentNames, len(tc.expectedDashParentNames))
require.ElementsMatch(t, resDashParentNames, tc.expectedDashParentNames)
resFoldParentNames := slices.Collect(maps.Values(dataUIDsToParentNamesByType[cloudmigration.FolderDataType]))
require.Len(t, resFoldParentNames, len(tc.expectedFoldParentNames))
require.ElementsMatch(t, resFoldParentNames, tc.expectedFoldParentNames)
}
}
func TestGetLibraryElementsCommands(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
pkg/services/cloudmigration/cloudmigrationimpl/fake/cloudmigration_fake.go
+18
View File
@@ -98,10 +98,28 @@ func (m FakeServiceImpl) GetSnapshot(ctx context.Context, query cloudmigration.G
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
cloudMigrationResources := []cloudmigration.CloudMigrationResource{
{
Type: cloudmigration.DashboardDataType,
RefID: "123",
Status: cloudmigration.ItemStatusPending,
Name: "dashboard name",
ParentName: "dashboard parent name",
},
{
Type: cloudmigration.DatasourceDataType,
RefID: "456",
Status: cloudmigration.ItemStatusOK,
Name: "datasource name",
ParentName: "dashboard parent name",
},
}
return &cloudmigration.CloudMigrationSnapshot{
UID: "fake_uid",
SessionUID: "fake_uid",
Status: cloudmigration.SnapshotStatusCreating,
Resources: cloudMigrationResources,
}, nil
}
pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go
+93 -11
View File
@@ -27,6 +27,13 @@ import (
"go.opentelemetry.io/otel/codes"
)
var currentMigrationTypes = []cloudmigration.MigrateDataType{
cloudmigration.DatasourceDataType,
cloudmigration.FolderDataType,
cloudmigration.LibraryElementDataType,
cloudmigration.DashboardDataType,
}
func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.SignedInUser) (*cloudmigration.MigrateDataRequest, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.getMigrationDataJSON")
defer span.End()
@@ -100,8 +107,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Obtain the names of parent elements for Dashboard and Folders data types
parentNamesByType, err := s.getParentNames(ctx, signedInUser, dashs, folders, libraryElements)
if err != nil {
s.log.Error("Failed to get parent folder names", "err", err)
}
migrationData := &cloudmigration.MigrateDataRequest{
Items: migrationDataSlice,
Items: migrationDataSlice,
ItemParentNames: parentNamesByType,
}
return migrationData, nil
@@ -306,20 +320,21 @@ func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedIn
Data: item.Data,
})
parentName := ""
if _, exists := migrationData.ItemParentNames[item.Type]; exists {
parentName = migrationData.ItemParentNames[item.Type][item.RefID]
}
localSnapshotResource[i] = cloudmigration.CloudMigrationResource{
Name: item.Name,
Type: item.Type,
RefID: item.RefID,
Status: cloudmigration.ItemStatusPending,
Name: item.Name,
Type: item.Type,
RefID: item.RefID,
Status: cloudmigration.ItemStatusPending,
ParentName: parentName,
}
}
for _, resourceType := range []cloudmigration.MigrateDataType{
cloudmigration.DatasourceDataType,
cloudmigration.FolderDataType,
cloudmigration.LibraryElementDataType,
cloudmigration.DashboardDataType,
} {
for _, resourceType := range currentMigrationTypes {
for chunk := range slices.Chunk(resourcesGroupedByType[resourceType], int(maxItemsPerPartition)) {
if err := snapshotWriter.Write(string(resourceType), chunk); err != nil {
return fmt.Errorf("writing resources to snapshot writer: resourceType=%s %w", resourceType, err)
@@ -533,3 +548,70 @@ func sortFolders(input []folder.CreateFolderCommand) []folder.CreateFolderComman
return input
}
// getFolderNamesForFolderUIDs queries the folders service to obtain folder names for a list of folderUIDs
func (s *Service) getFolderNamesForFolderUIDs(ctx context.Context, signedInUser *user.SignedInUser, folderUIDs []string) (map[string](string), error) {
folders, err := s.folderService.GetFolders(ctx, folder.GetFoldersQuery{
UIDs: folderUIDs,
SignedInUser: signedInUser,
WithFullpathUIDs: true,
})
if err != nil {
s.log.Error("Failed to obtain folders from folder UIDs", "err", err)
return nil, err
}
folderUIDsToNames := make(map[string](string), len(folderUIDs))
for _, folderUID := range folderUIDs {
folderUIDsToNames[folderUID] = ""
}
for _, f := range folders {
folderUIDsToNames[f.UID] = f.Title
}
return folderUIDsToNames, nil
}
// getParentNames finds the parent names for resources and returns a map of data type: {data UID : parentName}
// for dashboards, folders and library elements - the parent is the parent folder
func (s *Service) getParentNames(ctx context.Context, signedInUser *user.SignedInUser, dashboards []dashboards.Dashboard, folders []folder.CreateFolderCommand, libraryElements []libraryElement) (map[cloudmigration.MigrateDataType]map[string](string), error) {
parentNamesByType := make(map[cloudmigration.MigrateDataType]map[string](string))
for _, dataType := range currentMigrationTypes {
parentNamesByType[dataType] = make(map[string]string)
}
// Obtain list of unique folderUIDs
parentFolderUIDsSet := make(map[string]struct{}, len(dashboards)+len(folders)+len(libraryElements))
for _, dashboard := range dashboards {
parentFolderUIDsSet[dashboard.FolderUID] = struct{}{}
}
for _, f := range folders {
parentFolderUIDsSet[f.ParentUID] = struct{}{}
}
for _, libraryElement := range libraryElements {
parentFolderUIDsSet[*libraryElement.FolderUID] = struct{}{}
}
parentFolderUIDsSlice := make([]string, 0, len(parentFolderUIDsSet))
for parentFolderUID := range parentFolderUIDsSet {
parentFolderUIDsSlice = append(parentFolderUIDsSlice, parentFolderUID)
}
// Obtain folder names given a list of folderUIDs
foldersUIDsToFolderName, err := s.getFolderNamesForFolderUIDs(ctx, signedInUser, parentFolderUIDsSlice)
if err != nil {
s.log.Error("Failed to get parent folder names from folder UIDs", "err", err)
return parentNamesByType, err
}
// Prepare map of {data type: {data UID : parentName}}
for _, dashboard := range dashboards {
parentNamesByType[cloudmigration.DashboardDataType][dashboard.UID] = foldersUIDsToFolderName[dashboard.FolderUID]
}
for _, f := range folders {
parentNamesByType[cloudmigration.FolderDataType][f.UID] = foldersUIDsToFolderName[f.ParentUID]
}
for _, libraryElement := range libraryElements {
parentNamesByType[cloudmigration.LibraryElementDataType][libraryElement.UID] = foldersUIDsToFolderName[*libraryElement.FolderUID]
}
return parentNamesByType, err
}