public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CloudMigrations: Add support for migration of Library Elements (Panels) resources (#93898)

Browse Source 
* CloudMigrations: create snapshots of Library Elements

* CloudMigrations: render library element resource in resources table

* CloudMigrations: create newtype with necessary fields for library element creation
This commit is contained in:
Matheus Macabu
2024-10-03 11:54:54 +02:00
committed by GitHub
parent dd7f45011d
commit 1635a3cd67
14 changed files with 292 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go
+23 -19
View File
@@ -26,6 +26,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/gcom"
"github.com/grafana/grafana/pkg/services/libraryelements"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
"github.com/grafana/grafana/pkg/services/secrets"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -53,13 +54,14 @@ type Service struct {
gmsClient gmsclient.Client
objectStorage objectstorage.ObjectStorage
dsService datasources.DataSourceService
gcomService gcom.Service
dashboardService dashboards.DashboardService
folderService folder.Service
pluginStore pluginstore.Store
secretsService secrets.Service
kvStore *kvstore.NamespacedKVStore
dsService datasources.DataSourceService
gcomService gcom.Service
dashboardService dashboards.DashboardService
folderService folder.Service
pluginStore pluginstore.Store
secretsService secrets.Service
kvStore *kvstore.NamespacedKVStore
libraryElementsService libraryelements.Service
api *api.CloudMigrationAPI
tracer tracing.Tracer
@@ -93,24 +95,26 @@ func ProvideService(
folderService folder.Service,
pluginStore pluginstore.Store,
kvStore kvstore.KVStore,
libraryElementsService libraryelements.Service,
) (cloudmigration.Service, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrations) {
return &NoopServiceImpl{}, nil
}
s := &Service{
store: &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
log: log.New(LogPrefix),
cfg: cfg,
features: features,
dsService: dsService,
tracer: tracer,
metrics: newMetrics(),
secretsService: secretsService,
dashboardService: dashboardService,
folderService: folderService,
pluginStore: pluginStore,
kvStore: kvstore.WithNamespace(kvStore, 0, "cloudmigration"),
store: &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
log: log.New(LogPrefix),
cfg: cfg,
features: features,
dsService: dsService,
tracer: tracer,
metrics: newMetrics(),
secretsService: secretsService,
dashboardService: dashboardService,
folderService: folderService,
pluginStore: pluginStore,
kvStore: kvstore.WithNamespace(kvStore, 0, "cloudmigration"),
libraryElementsService: libraryElementsService,
}
s.api = api.RegisterApi(routeRegister, s, tracer)
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
+33
View File
@@ -24,6 +24,8 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/folder/foldertest"
libraryelementsfake "github.com/grafana/grafana/pkg/services/libraryelements/fake"
libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -562,6 +564,36 @@ func TestReportEvent(t *testing.T) {
})
}
func TestGetLibraryElementsCommands(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
libraryElementService, ok := s.libraryElementsService.(*libraryelementsfake.LibraryElementService)
require.True(t, ok)
require.NotNil(t, libraryElementService)
folderUID := "folder-uid"
createLibraryElementCmd := libraryelements.CreateLibraryElementCommand{
FolderUID: &folderUID,
Name: "library-element-1",
Model: []byte{},
Kind: int64(libraryelements.PanelElement),
UID: "library-element-uid-1",
}
user := &user.SignedInUser{OrgID: 1}
_, err := libraryElementService.CreateElement(ctx, user, createLibraryElementCmd)
require.NoError(t, err)
cmds, err := s.getLibraryElementsCommands(ctx, user)
require.NoError(t, err)
require.Len(t, cmds, 1)
require.Equal(t, createLibraryElementCmd.UID, cmds[0].UID)
}
func ctxWithSignedInUser() context.Context {
c := &contextmodel.ReqContext{
SignedInUser: &user.SignedInUser{OrgID: 1},
@@ -626,6 +658,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi
mockFolder,
&pluginstore.FakePluginStore{},
kvstore.ProvideService(sqlStore),
&libraryelementsfake.LibraryElementService{},
)
require.NoError(t, err)
pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go
+73 -1
View File
@@ -3,6 +3,7 @@ package cloudmigrationimpl
import (
"context"
cryptoRand "crypto/rand"
"encoding/json"
"fmt"
"os"
"path/filepath"
@@ -18,6 +19,7 @@ import (
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/util/retryer"
"golang.org/x/crypto/nacl/box"
@@ -38,9 +40,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
return nil, err
}
libraryElements, err := s.getLibraryElementsCommands(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get library elements", "err", err)
return nil, err
}
migrationDataSlice := make(
[]cloudmigration.MigrateDataRequestItem, 0,
len(dataSources)+len(dashs)+len(folders),
len(dataSources)+len(dashs)+len(folders)+len(libraryElements),
)
for _, ds := range dataSources {
@@ -78,6 +86,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
for _, libraryElement := range libraryElements {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.LibraryElementDataType,
RefID: libraryElement.UID,
Name: libraryElement.Name,
Data: libraryElement,
})
}
migrationData := &cloudmigration.MigrateDataRequest{
Items: migrationDataSlice,
}
@@ -169,6 +186,60 @@ func (s *Service) getDashboardAndFolderCommands(ctx context.Context, signedInUse
return dashboardCmds, folderCmds, nil
}
type libraryElement struct {
FolderUID *string `json:"folderUid"`
Name string `json:"name"`
UID string `json:"uid"`
Model json.RawMessage `json:"model"`
Kind int64 `json:"kind"`
}
// getLibraryElementsCommands returns the json payloads required by the library elements creation API
func (s *Service) getLibraryElementsCommands(ctx context.Context, signedInUser *user.SignedInUser) ([]libraryElement, error) {
const perPage = 100
cmds := make([]libraryElement, 0)
page := 1
count := 0
for {
query := libraryelements.SearchLibraryElementsQuery{
PerPage: perPage,
Page: page,
}
libraryElements, err := s.libraryElementsService.GetAllElements(ctx, signedInUser, query)
if err != nil {
return nil, fmt.Errorf("failed to get all library elements: %w", err)
}
for _, element := range libraryElements.Elements {
var folderUID *string
if len(element.FolderUID) > 0 {
folderUID = &element.FolderUID
}
cmds = append(cmds, libraryElement{
FolderUID: folderUID,
Name: element.Name,
Model: element.Model,
Kind: element.Kind,
UID: element.UID,
})
}
page += 1
count += libraryElements.PerPage
if len(libraryElements.Elements) == 0 || count >= int(libraryElements.TotalCount) {
break
}
}
return cmds, nil
}
// asynchronous process for writing the snapshot to the filesystem and updating the snapshot status
func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedInUser, maxItemsPerPartition uint32, metadata []byte, snapshotMeta cloudmigration.CloudMigrationSnapshot) error {
// TODO -- make sure we can only build one snapshot at a time
@@ -229,6 +300,7 @@ func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedIn
for _, resourceType := range []cloudmigration.MigrateDataType{
cloudmigration.DatasourceDataType,
cloudmigration.FolderDataType,
cloudmigration.LibraryElementDataType,
cloudmigration.DashboardDataType,
} {
for chunk := range slices.Chunk(resourcesGroupedByType[resourceType], int(maxItemsPerPartition)) {