public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CloudMigrations: Implement snapshot management apis (#89296)

Browse Source 
* add new apis

* add payloads

* create snapshot status type

* add some impl

* finish implementing update

* start implementing build snapshot func

* add more fake build logic

* add cancel endpoint. do some cleanup

* implement GetSnapshot

* implement upload snapshot

* merge onprem status with gms result

* get it working

* update comment

* rename list endpoint

* add query limit and offset

* add helper method to snapshot

* little bit of cleanup

* work on swagger annotations

* manual merge

* generate swagger specs

* clean up curl commands

* fix bugs found during final testing

* fix linter issue

* fix unit test
This commit is contained in:
Michael Mandrus
2024-06-19 09:20:52 -04:00
committed by GitHub
parent d928fac5c3
commit 8a8f97b0e4
20 changed files with 1939 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/api/api.go
+212 -5
View File
@@ -40,19 +40,28 @@ func RegisterApi(
// registerEndpoints Registers Endpoints on Grafana Router
func (cma *CloudMigrationAPI) registerEndpoints() {
cma.routeRegister.Group("/api/cloudmigration", func(cloudMigrationRoute routing.RouteRegister) {
// destination instance endpoints for token management
cloudMigrationRoute.Get("/token", routing.Wrap(cma.GetToken))
cloudMigrationRoute.Post("/token", routing.Wrap(cma.CreateToken))
cloudMigrationRoute.Delete("/token/:uid", routing.Wrap(cma.DeleteToken))
// on-prem instance endpoints for managing GMS sessions
cloudMigrationRoute.Get("/migration", routing.Wrap(cma.GetSessionList))
cloudMigrationRoute.Post("/migration", routing.Wrap(cma.CreateSession))
cloudMigrationRoute.Get("/migration/:uid", routing.Wrap(cma.GetSession))
cloudMigrationRoute.Delete("/migration/:uid", routing.Wrap(cma.DeleteSession))
// TODO new APIs for snapshot management to replace these
// sync approach to data migration
cloudMigrationRoute.Post("/migration/:uid/run", routing.Wrap(cma.RunMigration))
cloudMigrationRoute.Get("/migration/:uid/run", routing.Wrap(cma.GetMigrationRunList))
cloudMigrationRoute.Get("/migration/run/:runUID", routing.Wrap(cma.GetMigrationRun))
cloudMigrationRoute.Get("/token", routing.Wrap(cma.GetToken))
cloudMigrationRoute.Post("/token", routing.Wrap(cma.CreateToken))
cloudMigrationRoute.Delete("/token/:uid", routing.Wrap(cma.DeleteToken))
// async approach to data migration using snapshots
cloudMigrationRoute.Post("/migration/:uid/snapshot", routing.Wrap(cma.CreateSnapshot))
cloudMigrationRoute.Get("/migration/:uid/snapshot/:snapshotUid", routing.Wrap(cma.GetSnapshot))
cloudMigrationRoute.Get("/migration/:uid/snapshots", routing.Wrap(cma.GetSnapshotList))
cloudMigrationRoute.Post("/migration/:uid/snapshot/:snapshotUid/upload", routing.Wrap(cma.UploadSnapshot))
cloudMigrationRoute.Post("/migration/:uid/snapshot/:snapshotUid/cancel", routing.Wrap(cma.CancelSnapshot))
}, middleware.ReqOrgAdmin)
}
@@ -121,6 +130,7 @@ func (cma *CloudMigrationAPI) CreateToken(c *contextmodel.ReqContext) response.R
//
// Responses:
// 204: cloudMigrationDeleteTokenResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -170,6 +180,7 @@ func (cma *CloudMigrationAPI) GetSessionList(c *contextmodel.ReqContext) respons
//
// Responses:
// 200: cloudMigrationSessionResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -201,6 +212,7 @@ func (cma *CloudMigrationAPI) GetSession(c *contextmodel.ReqContext) response.Re
//
// Responses:
// 200: cloudMigrationSessionResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -235,6 +247,7 @@ func (cma *CloudMigrationAPI) CreateSession(c *contextmodel.ReqContext) response
//
// Responses:
// 200: cloudMigrationRunResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -261,6 +274,7 @@ func (cma *CloudMigrationAPI) RunMigration(c *contextmodel.ReqContext) response.
//
// Responses:
// 200: cloudMigrationRunResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -293,6 +307,7 @@ func (cma *CloudMigrationAPI) GetMigrationRun(c *contextmodel.ReqContext) respon
//
// Responses:
// 200: cloudMigrationRunListResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
@@ -314,7 +329,7 @@ func (cma *CloudMigrationAPI) GetMigrationRunList(c *contextmodel.ReqContext) re
for i := 0; i < len(runList.Runs); i++ {
runs[i] = MigrateDataResponseListDTO{runList.Runs[i].RunUID}
}
return response.JSON(http.StatusOK, SnapshotListDTO{
return response.JSON(http.StatusOK, CloudMigrationRunListDTO{
Runs: runs,
})
}
@@ -326,6 +341,7 @@ func (cma *CloudMigrationAPI) GetMigrationRunList(c *contextmodel.ReqContext) re
// Responses:
// 200
// 401: unauthorisedError
// 400: badRequestError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) DeleteSession(c *contextmodel.ReqContext) response.Response {
@@ -343,3 +359,194 @@ func (cma *CloudMigrationAPI) DeleteSession(c *contextmodel.ReqContext) response
}
return response.Empty(http.StatusOK)
}
// swagger:route POST /cloudmigration/migration/{uid}/snapshot migrations createSnapshot
//
// Trigger the creation of an instance snapshot associated with the provided session.
// If the snapshot initialization is successful, the snapshot uid is returned.
//
// Responses:
// 200: createSnapshotResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) CreateSnapshot(c *contextmodel.ReqContext) response.Response {
ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.CreateSnapshot")
defer span.End()
uid := web.Params(c.Req)[":uid"]
if err := util.ValidateUID(uid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
}
ss, err := cma.cloudMigrationService.CreateSnapshot(ctx, uid)
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "error creating snapshot", err)
}
return response.JSON(http.StatusOK, CreateSnapshotResponseDTO{
SnapshotUID: ss.UID,
})
}
// swagger:route GET /cloudmigration/migration/{uid}/snapshot/{snapshotUid} migrations getSnapshot
//
// Get metadata about a snapshot, including where it is in its processing and final results.
//
// Responses:
// 200: getSnapshotResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.Response {
ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.GetSnapshot")
defer span.End()
sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
if err := util.ValidateUID(sessUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
}
if err := util.ValidateUID(snapshotUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
}
snapshot, err := cma.cloudMigrationService.GetSnapshot(ctx, sessUid, snapshotUid)
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "error retrieving snapshot", err)
}
result, err := snapshot.GetSnapshotResult()
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "error snapshot reading snapshot results", err)
}
dtoResults := make([]MigrateDataResponseItemDTO, len(result))
for i := 0; i < len(result); i++ {
dtoResults[i] = MigrateDataResponseItemDTO{
Type: MigrateDataType(result[i].Type),
RefID: result[i].RefID,
Status: ItemStatus(result[i].Status),
Error: result[i].Error,
}
}
respDto := GetSnapshotResponseDTO{
SnapshotDTO: SnapshotDTO{
SnapshotUID: snapshot.UID,
Status: fromSnapshotStatus(snapshot.Status),
SessionUID: sessUid,
Created: snapshot.Created,
Finished: snapshot.Finished,
},
Results: dtoResults,
}
return response.JSON(http.StatusOK, respDto)
}
// swagger:route GET /cloudmigration/migration/{uid}/snapshots migrations getShapshotList
//
// Get a list of snapshots for a session.
//
// Responses:
// 200: snapshotListResponse
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) GetSnapshotList(c *contextmodel.ReqContext) response.Response {
ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.GetShapshotList")
defer span.End()
uid := web.Params(c.Req)[":uid"]
if err := util.ValidateUID(uid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
}
q := cloudmigration.ListSnapshotsQuery{
SessionUID: uid,
Limit: c.QueryInt("limit"),
Offset: c.QueryInt("offset"),
}
if q.Limit == 0 {
q.Limit = 100
}
snapshotList, err := cma.cloudMigrationService.GetSnapshotList(ctx, q)
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "error retrieving snapshot list", err)
}
dtos := make([]SnapshotDTO, len(snapshotList))
for i := 0; i < len(snapshotList); i++ {
dtos[i] = SnapshotDTO{
SnapshotUID: snapshotList[i].UID,
Status: fromSnapshotStatus(snapshotList[i].Status),
SessionUID: uid,
Created: snapshotList[i].Created,
Finished: snapshotList[i].Finished,
}
}
return response.JSON(http.StatusOK, SnapshotListResponseDTO{
Snapshots: dtos,
})
}
// swagger:route POST /cloudmigration/migration/{uid}/snapshot/{snapshotUid}/upload migrations uploadSnapshot
//
// Upload a snapshot to the Grafana Migration Service for processing.
//
// Responses:
// 200:
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) UploadSnapshot(c *contextmodel.ReqContext) response.Response {
ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.UploadSnapshot")
defer span.End()
sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
if err := util.ValidateUID(sessUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
}
if err := util.ValidateUID(snapshotUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
}
if err := cma.cloudMigrationService.UploadSnapshot(ctx, sessUid, snapshotUid); err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "error uploading snapshot", err)
}
return response.JSON(http.StatusOK, nil)
}
// swagger:route POST /cloudmigration/migration/{uid}/snapshot/{snapshotUid}/cancel migrations cancelSnapshot
//
// Cancel a snapshot, wherever it is in its processing chain.
// TODO: Implement
//
// Responses:
// 200:
// 400: badRequestError
// 401: unauthorisedError
// 403: forbiddenError
// 500: internalServerError
func (cma *CloudMigrationAPI) CancelSnapshot(c *contextmodel.ReqContext) response.Response {
_, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.CancelSnapshot")
defer span.End()
sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
if err := util.ValidateUID(sessUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
}
if err := util.ValidateUID(snapshotUid); err != nil {
return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
}
// Implement
return response.JSON(http.StatusOK, nil)
}
pkg/services/cloudmigration/api/curl_commands.txt
+21
View File
@@ -0,0 +1,21 @@
[sample token] // NOT A REAL TOKEN
eyJUb2tlbiI6ImNvbXBsZXRlbHlfZmFrZV90b2tlbl9jZG9peTFhYzdwdXlwZCIsIkluc3RhbmNlIjp7IlN0YWNrSUQiOjEyMzQ1LCJTbHVnIjoic3R1Ymluc3RhbmNlIiwiUmVnaW9uU2x1ZyI6ImZha2UtcmVnaW9uIiwiQ2x1c3RlclNsdWciOiJmYWtlLWNsdXNlciJ9fQ==
[create session}
curl -X POST -H "Content-Type: application/json" \
http://admin:admin@localhost:3000/api/cloudmigration/migration \
-d '{"AuthToken":"eyJUb2tlbiI6ImNvbXBsZXRlbHlfZmFrZV90b2tlbl9jZG9peTFhYzdwdXlwZCIsIkluc3RhbmNlIjp7IlN0YWNrSUQiOjEyMzQ1LCJTbHVnIjoic3R1Ymluc3RhbmNlIiwiUmVnaW9uU2x1ZyI6ImZha2UtcmVnaW9uIiwiQ2x1c3RlclNsdWciOiJmYWtlLWNsdXNlciJ9fQ=="}'
[create snapshot]
curl -X POST -H "Content-Type: application/json" \
http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot
[get snapshot list]
curl -X GET http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshots?limit=100&offset=0
[get snapshot]
curl -X GET http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot/{snapshotUid}
[upload snapshot]
curl -X POST -H "Content-Type: application/json" \
http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot/{snapshotUid}/upload
pkg/services/cloudmigration/api/dtos.go
+137 -8
View File
@@ -38,7 +38,6 @@ type CreateAccessTokenResponseDTO struct {
// swagger:parameters deleteCloudMigrationToken
type DeleteCloudMigrationToken struct {
// UID of a cloud migration token
//
// in: path
UID string `json:"uid"`
}
@@ -67,7 +66,6 @@ type CloudMigrationSessionListResponseDTO struct {
// swagger:parameters getSession
type GetCloudMigrationSessionRequest struct {
// UID of a migration session
//
// in: path
UID string `json:"uid"`
}
@@ -92,7 +90,6 @@ type CloudMigrationSessionRequestDTO struct {
// swagger:parameters runCloudMigration
type RunCloudMigrationRequest struct {
// UID of a migration
//
// in: path
UID string `json:"uid"`
}
@@ -138,7 +135,6 @@ const (
// swagger:parameters getCloudMigrationRun
type GetMigrationRunParams struct {
// RunUID of a migration run
//
// in: path
RunUID string `json:"runUID"`
}
@@ -146,7 +142,6 @@ type GetMigrationRunParams struct {
// swagger:parameters getCloudMigrationRunList
type GetCloudMigrationRunList struct {
// UID of a migration
//
// in: path
UID string `json:"uid"`
}
@@ -154,10 +149,10 @@ type GetCloudMigrationRunList struct {
// swagger:response cloudMigrationRunListResponse
type CloudMigrationRunListResponse struct {
// in: body
Body SnapshotListDTO
Body CloudMigrationRunListDTO
}
type SnapshotListDTO struct {
type CloudMigrationRunListDTO struct {
Runs []MigrateDataResponseListDTO `json:"runs"`
}
@@ -168,7 +163,6 @@ type MigrateDataResponseListDTO struct {
// swagger:parameters deleteSession
type DeleteMigrationSessionRequest struct {
// UID of a migration session
//
// in: path
UID string `json:"uid"`
}
@@ -207,3 +201,138 @@ func convertMigrateDataResponseToDTO(r cloudmigration.MigrateDataResponse) Migra
Items: items,
}
}
// Base snapshot without results
type SnapshotDTO struct {
SnapshotUID string `json:"uid"`
Status SnapshotStatus `json:"status"`
SessionUID string `json:"sessionUid"`
Created time.Time `json:"created"`
Finished time.Time `json:"finished"`
}
// swagger:enum SnapshotStatus
type SnapshotStatus string
const (
SnapshotStatusInitializing SnapshotStatus = "INITIALIZING"
SnapshotStatusCreating SnapshotStatus = "CREATING"
SnapshotStatusPendingUpload SnapshotStatus = "PENDING_UPLOAD"
SnapshotStatusUploading SnapshotStatus = "UPLOADING"
SnapshotStatusPendingProcessing SnapshotStatus = "PENDING_PROCESSING"
SnapshotStatusProcessing SnapshotStatus = "PROCESSING"
SnapshotStatusFinished SnapshotStatus = "FINISHED"
SnapshotStatusError SnapshotStatus = "ERROR"
SnapshotStatusUnknown SnapshotStatus = "UNKNOWN"
)
func fromSnapshotStatus(status cloudmigration.SnapshotStatus) SnapshotStatus {
switch status {
case cloudmigration.SnapshotStatusInitializing:
return SnapshotStatusInitializing
case cloudmigration.SnapshotStatusCreating:
return SnapshotStatusCreating
case cloudmigration.SnapshotStatusPendingUpload:
return SnapshotStatusPendingUpload
case cloudmigration.SnapshotStatusUploading:
return SnapshotStatusUploading
case cloudmigration.SnapshotStatusPendingProcessing:
return SnapshotStatusPendingProcessing
case cloudmigration.SnapshotStatusProcessing:
return SnapshotStatusProcessing
case cloudmigration.SnapshotStatusFinished:
return SnapshotStatusFinished
case cloudmigration.SnapshotStatusError:
return SnapshotStatusError
default:
return SnapshotStatusUnknown
}
}
// swagger:parameters createSnapshot
type CreateSnapshotRequest struct {
// UID of a session
// in: path
UID string `json:"uid"`
}
// swagger:response createSnapshotResponse
type CreateSnapshotResponse struct {
// in: body
Body CreateSnapshotResponseDTO
}
type CreateSnapshotResponseDTO struct {
SnapshotUID string `json:"uid"`
}
// swagger:parameters getSnapshot
type GetSnapshotParams struct {
// Session UID of a session
// in: path
UID string `json:"uid"`
// UID of a snapshot
// in: path
SnapshotUID string `json:"snapshotUid"`
}
// swagger:response getSnapshotResponse
type GetSnapshotResponse struct {
// in: body
Body GetSnapshotResponseDTO
}
type GetSnapshotResponseDTO struct {
SnapshotDTO
Results []MigrateDataResponseItemDTO `json:"results"`
}
// swagger:parameters getShapshotList
type GetSnapshotListParams struct {
// Offset is used for pagination with limit
// in:query
// required:false
// default: 0
Offset int `json:"offset"`
// Max limit for results returned.
// in:query
// required:false
// default: 100
Limit int `json:"limit"`
// Session UID of a session
// in: path
UID string `json:"uid"`
}
// swagger:response snapshotListResponse
type SnapshotListResponse struct {
// in: body
Body SnapshotListResponseDTO
}
type SnapshotListResponseDTO struct {
Snapshots []SnapshotDTO `json:"snapshots"`
}
// swagger:parameters uploadSnapshot
type UploadSnapshotParams struct {
// Session UID of a session
// in: path
UID string `json:"uid"`
// UID of a snapshot
// in: path
SnapshotUID string `json:"snapshotUid"`
}
// swagger:parameters cancelSnapshot
type CancelSnapshotParams struct {
// Session UID of a session
// in: path
UID string `json:"uid"`
// UID of a snapshot
// in: path
SnapshotUID string `json:"snapshotUid"`
}
pkg/services/cloudmigration/cloudmigration.go
+6 -1
View File
@@ -22,5 +22,10 @@ type Service interface {
RunMigration(ctx context.Context, migUID string) (*MigrateDataResponse, error)
GetMigrationStatus(ctx context.Context, runUID string) (*CloudMigrationSnapshot, error)
GetMigrationRunList(ctx context.Context, migUID string) (*SnapshotList, error)
GetMigrationRunList(ctx context.Context, migUID string) (*CloudMigrationRunList, error)
CreateSnapshot(ctx context.Context, sessionUid string) (*CloudMigrationSnapshot, error)
GetSnapshot(ctx context.Context, sessionUid string, snapshotUid string) (*CloudMigrationSnapshot, error)
GetSnapshotList(ctx context.Context, query ListSnapshotsQuery) ([]CloudMigrationSnapshot, error)
UploadSnapshot(ctx context.Context, sessionUid string, snapshotUid string) error
}
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go
+126 -133
View File
@@ -7,6 +7,9 @@ import (
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/grafana/grafana/pkg/api/response"
@@ -17,7 +20,6 @@ import (
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/cloudmigration/api"
"github.com/grafana/grafana/pkg/services/cloudmigration/gmsclient"
"github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -25,6 +27,7 @@ import (
"github.com/grafana/grafana/pkg/services/gcom"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
@@ -37,6 +40,9 @@ type Service struct {
log *log.ConcreteLogger
cfg *setting.Cfg
buildSnapshotMutex sync.Mutex
buildSnapshotError bool
features featuremgmt.FeatureToggles
gmsClient gmsclient.Client
@@ -398,7 +404,6 @@ func (s *Service) RunMigration(ctx context.Context, uid string) (*cloudmigration
return nil, fmt.Errorf("migrate data error: %w", err)
}
// TODO update cloud migration run schema to treat the result as a first-class citizen
respData, err := json.Marshal(resp)
if err != nil {
s.log.Error("error marshalling migration response data: %w", err)
@@ -419,135 +424,6 @@ func (s *Service) RunMigration(ctx context.Context, uid string) (*cloudmigration
return resp, nil
}
func (s *Service) getMigrationDataJSON(ctx context.Context) (*cloudmigration.MigrateDataRequest, error) {
// Data sources
dataSources, err := s.getDataSources(ctx)
if err != nil {
s.log.Error("Failed to get datasources", "err", err)
return nil, err
}
// Dashboards
dashboards, err := s.getDashboards(ctx)
if err != nil {
s.log.Error("Failed to get dashboards", "err", err)
return nil, err
}
// Folders
folders, err := s.getFolders(ctx)
if err != nil {
s.log.Error("Failed to get folders", "err", err)
return nil, err
}
migrationDataSlice := make(
[]cloudmigration.MigrateDataRequestItem, 0,
len(dataSources)+len(dashboards)+len(folders),
)
for _, ds := range dataSources {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DatasourceDataType,
RefID: ds.UID,
Name: ds.Name,
Data: ds,
})
}
for _, dashboard := range dashboards {
dashboard.Data.Del("id")
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DashboardDataType,
RefID: dashboard.UID,
Name: dashboard.Title,
Data: map[string]any{"dashboard": dashboard.Data},
})
}
for _, f := range folders {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.FolderDataType,
RefID: f.UID,
Name: f.Title,
Data: f,
})
}
migrationData := &cloudmigration.MigrateDataRequest{
Items: migrationDataSlice,
}
return migrationData, nil
}
func (s *Service) getDataSources(ctx context.Context) ([]datasources.AddDataSourceCommand, error) {
dataSources, err := s.dsService.GetAllDataSources(ctx, &datasources.GetAllDataSourcesQuery{})
if err != nil {
s.log.Error("Failed to get all datasources", "err", err)
return nil, err
}
result := []datasources.AddDataSourceCommand{}
for _, dataSource := range dataSources {
// Decrypt secure json to send raw credentials
decryptedData, err := s.secretsService.DecryptJsonData(ctx, dataSource.SecureJsonData)
if err != nil {
s.log.Error("Failed to decrypt secure json data", "err", err)
return nil, err
}
dataSourceCmd := datasources.AddDataSourceCommand{
OrgID: dataSource.OrgID,
Name: dataSource.Name,
Type: dataSource.Type,
Access: dataSource.Access,
URL: dataSource.URL,
User: dataSource.User,
Database: dataSource.Database,
BasicAuth: dataSource.BasicAuth,
BasicAuthUser: dataSource.BasicAuthUser,
WithCredentials: dataSource.WithCredentials,
IsDefault: dataSource.IsDefault,
JsonData: dataSource.JsonData,
SecureJsonData: decryptedData,
ReadOnly: dataSource.ReadOnly,
UID: dataSource.UID,
}
result = append(result, dataSourceCmd)
}
return result, err
}
func (s *Service) getFolders(ctx context.Context) ([]folder.Folder, error) {
reqCtx := contexthandler.FromContext(ctx)
folders, err := s.folderService.GetFolders(ctx, folder.GetFoldersQuery{
SignedInUser: reqCtx.SignedInUser,
})
if err != nil {
return nil, err
}
result := make([]folder.Folder, len(folders))
for i, folder := range folders {
result[i] = *folder
}
return result, nil
}
func (s *Service) getDashboards(ctx context.Context) ([]dashboards.Dashboard, error) {
dashs, err := s.dashboardService.GetAllDashboards(ctx)
if err != nil {
return nil, err
}
result := make([]dashboards.Dashboard, len(dashs))
for i, dashboard := range dashs {
result[i] = *dashboard
}
return result, nil
}
func (s *Service) createMigrationRun(ctx context.Context, cmr cloudmigration.CloudMigrationSnapshot) (string, error) {
uid, err := s.store.CreateMigrationRun(ctx, cmr)
if err != nil {
@@ -565,13 +441,13 @@ func (s *Service) GetMigrationStatus(ctx context.Context, runUID string) (*cloud
return cmr, nil
}
func (s *Service) GetMigrationRunList(ctx context.Context, migUID string) (*cloudmigration.SnapshotList, error) {
func (s *Service) GetMigrationRunList(ctx context.Context, migUID string) (*cloudmigration.CloudMigrationRunList, error) {
runs, err := s.store.GetMigrationStatusList(ctx, migUID)
if err != nil {
return nil, fmt.Errorf("retrieving migration statuses from db: %w", err)
}
runList := &cloudmigration.SnapshotList{Runs: []cloudmigration.MigrateDataResponseList{}}
runList := &cloudmigration.CloudMigrationRunList{Runs: []cloudmigration.MigrateDataResponseList{}}
for _, s := range runs {
runList.Runs = append(runList.Runs, cloudmigration.MigrateDataResponseList{
RunUID: s.UID,
@@ -589,6 +465,123 @@ func (s *Service) DeleteSession(ctx context.Context, uid string) (*cloudmigratio
return c, nil
}
func (s *Service) CreateSnapshot(ctx context.Context, sessionUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.CreateSnapshot")
defer span.End()
// fetch session for the gms auth token
session, err := s.store.GetMigrationSessionByUID(ctx, sessionUid)
if err != nil {
return nil, fmt.Errorf("fetching migration session for uid %s: %w", sessionUid, err)
}
// query gms to establish new snapshot
initResp, err := s.gmsClient.InitializeSnapshot(ctx, *session)
if err != nil {
return nil, fmt.Errorf("initializing snapshot with GMS for session %s: %w", sessionUid, err)
}
// create new directory for snapshot writing
snapshotUid := util.GenerateShortUID()
dir := filepath.Join("cloudmigration.snapshots", fmt.Sprintf("snapshot-%s-%s", snapshotUid, initResp.GMSSnapshotUID))
err = os.MkdirAll(dir, 0750)
if err != nil {
return nil, fmt.Errorf("creating snapshot directory: %w", err)
}
// save snapshot to the db
snapshot := cloudmigration.CloudMigrationSnapshot{
UID: snapshotUid,
SessionUID: sessionUid,
Status: cloudmigration.SnapshotStatusInitializing,
EncryptionKey: initResp.EncryptionKey,
UploadURL: initResp.UploadURL,
GMSSnapshotUID: initResp.GMSSnapshotUID,
LocalDir: dir,
}
uid, err := s.store.CreateSnapshot(ctx, snapshot)
if err != nil {
return nil, fmt.Errorf("saving snapshot: %w", err)
}
snapshot.UID = uid
// start building the snapshot asynchronously while we return a success response to the client
go s.buildSnapshot(context.Background(), snapshot)
return &snapshot, nil
}
// GetSnapshot returns the on-prem version of a snapshot, supplemented with processing status from GMS
func (s *Service) GetSnapshot(ctx context.Context, sessionUid string, snapshotUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.GetSnapshot")
defer span.End()
snapshot, err := s.store.GetSnapshotByUID(ctx, snapshotUid)
if err != nil {
return nil, fmt.Errorf("fetching snapshot for uid %s: %w", snapshotUid, err)
}
session, err := s.store.GetMigrationSessionByUID(ctx, sessionUid)
if err != nil {
return nil, fmt.Errorf("fetching session for uid %s: %w", sessionUid, err)
}
if snapshot.ShouldQueryGMS() {
// ask GMS for status if it's in the cloud
snapshotMeta, err := s.gmsClient.GetSnapshotStatus(ctx, *session, *snapshot)
if err != nil {
return nil, fmt.Errorf("error fetching snapshot status from GMS: sessionUid: %s, snapshotUid: %s", sessionUid, snapshotUid)
}
// grab any result available
// TODO: figure out a more intelligent way to do this, will depend on GMS apis
snapshot.Result = snapshotMeta.Result
if snapshotMeta.Status == cloudmigration.SnapshotStatusFinished {
// we need to update the snapshot in our db before reporting anything finished to the client
if err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshot.UID,
Status: cloudmigration.SnapshotStatusFinished,
Result: snapshot.Result,
}); err != nil {
return nil, fmt.Errorf("error updating snapshot status: %w", err)
}
}
}
return snapshot, nil
}
func (s *Service) GetSnapshotList(ctx context.Context, query cloudmigration.ListSnapshotsQuery) ([]cloudmigration.CloudMigrationSnapshot, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.GetSnapshotList")
defer span.End()
snapshotList, err := s.store.GetSnapshotList(ctx, query)
if err != nil {
return nil, fmt.Errorf("fetching snapshots for session uid %s: %w", query.SessionUID, err)
}
return snapshotList, nil
}
func (s *Service) UploadSnapshot(ctx context.Context, sessionUid string, snapshotUid string) error {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.UploadSnapshot")
defer span.End()
snapshot, err := s.GetSnapshot(ctx, sessionUid, snapshotUid)
if err != nil {
return fmt.Errorf("fetching snapshot with uid %s: %w", snapshotUid, err)
}
s.log.Info("Uploading snapshot with GMS ID %s in local directory %s to url %s", snapshot.GMSSnapshotUID, snapshot.LocalDir, snapshot.UploadURL)
s.log.Debug("UploadSnapshot not yet implemented, faking it")
// start uploading the snapshot asynchronously while we return a success response to the client
go s.uploadSnapshot(context.Background(), *snapshot)
return nil
}
func (s *Service) parseCloudMigrationConfig() (string, error) {
if s.cfg == nil {
return "", fmt.Errorf("cfg cannot be nil")
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_noop.go
+17 -1
View File
@@ -44,7 +44,7 @@ func (s *NoopServiceImpl) GetMigrationStatus(ctx context.Context, runUID string)
return nil, cloudmigration.ErrFeatureDisabledError
}
func (s *NoopServiceImpl) GetMigrationRunList(ctx context.Context, uid string) (*cloudmigration.SnapshotList, error) {
func (s *NoopServiceImpl) GetMigrationRunList(ctx context.Context, uid string) (*cloudmigration.CloudMigrationRunList, error) {
return nil, cloudmigration.ErrFeatureDisabledError
}
@@ -59,3 +59,19 @@ func (s *NoopServiceImpl) CreateMigrationRun(context.Context, cloudmigration.Clo
func (s *NoopServiceImpl) RunMigration(context.Context, string) (*cloudmigration.MigrateDataResponse, error) {
return nil, cloudmigration.ErrFeatureDisabledError
}
func (s *NoopServiceImpl) CreateSnapshot(ctx context.Context, sessionUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
return nil, cloudmigration.ErrFeatureDisabledError
}
func (s *NoopServiceImpl) GetSnapshot(ctx context.Context, sessionUid string, snapshotUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
return nil, cloudmigration.ErrFeatureDisabledError
}
func (s *NoopServiceImpl) GetSnapshotList(ctx context.Context, query cloudmigration.ListSnapshotsQuery) ([]cloudmigration.CloudMigrationSnapshot, error) {
return nil, cloudmigration.ErrFeatureDisabledError
}
func (s *NoopServiceImpl) UploadSnapshot(ctx context.Context, sessionUid string, snapshotUid string) error {
return cloudmigration.ErrFeatureDisabledError
}
pkg/services/cloudmigration/cloudmigrationimpl/fake/cloudmigration_fake.go
+50 -2
View File
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/gcom"
"github.com/grafana/grafana/pkg/util"
)
var fixedDate = time.Date(2024, 6, 5, 17, 30, 40, 0, time.UTC)
@@ -122,14 +123,61 @@ func (m FakeServiceImpl) GetMigrationStatus(_ context.Context, _ string) (*cloud
}, nil
}
func (m FakeServiceImpl) GetMigrationRunList(_ context.Context, _ string) (*cloudmigration.SnapshotList, error) {
func (m FakeServiceImpl) GetMigrationRunList(_ context.Context, _ string) (*cloudmigration.CloudMigrationRunList, error) {
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
return &cloudmigration.SnapshotList{
return &cloudmigration.CloudMigrationRunList{
Runs: []cloudmigration.MigrateDataResponseList{
{RunUID: "fake_run_uid_1"},
{RunUID: "fake_run_uid_2"},
},
}, nil
}
func (m FakeServiceImpl) CreateSnapshot(ctx context.Context, sessionUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
return &cloudmigration.CloudMigrationSnapshot{
UID: util.GenerateShortUID(),
SessionUID: sessionUid,
Status: cloudmigration.SnapshotStatusUnknown,
}, nil
}
func (m FakeServiceImpl) GetSnapshot(ctx context.Context, sessionUid string, snapshotUid string) (*cloudmigration.CloudMigrationSnapshot, error) {
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
return &cloudmigration.CloudMigrationSnapshot{
UID: util.GenerateShortUID(),
SessionUID: sessionUid,
Status: cloudmigration.SnapshotStatusUnknown,
}, nil
}
func (m FakeServiceImpl) GetSnapshotList(ctx context.Context, query cloudmigration.ListSnapshotsQuery) ([]cloudmigration.CloudMigrationSnapshot, error) {
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
return []cloudmigration.CloudMigrationSnapshot{
{
UID: util.GenerateShortUID(),
SessionUID: query.SessionUID,
Status: cloudmigration.SnapshotStatusUnknown,
},
{
UID: util.GenerateShortUID(),
SessionUID: query.SessionUID,
Status: cloudmigration.SnapshotStatusUnknown,
},
}, nil
}
func (m FakeServiceImpl) UploadSnapshot(ctx context.Context, sessionUid string, snapshotUid string) error {
if m.ReturnError {
return fmt.Errorf("mock error")
}
return nil
}
pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go
+234
View File
@@ -0,0 +1,234 @@
package cloudmigrationimpl
import (
"context"
"time"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/util/retryer"
)
func (s *Service) getMigrationDataJSON(ctx context.Context) (*cloudmigration.MigrateDataRequest, error) {
// Data sources
dataSources, err := s.getDataSources(ctx)
if err != nil {
s.log.Error("Failed to get datasources", "err", err)
return nil, err
}
// Dashboards
dashboards, err := s.getDashboards(ctx)
if err != nil {
s.log.Error("Failed to get dashboards", "err", err)
return nil, err
}
// Folders
folders, err := s.getFolders(ctx)
if err != nil {
s.log.Error("Failed to get folders", "err", err)
return nil, err
}
migrationDataSlice := make(
[]cloudmigration.MigrateDataRequestItem, 0,
len(dataSources)+len(dashboards)+len(folders),
)
for _, ds := range dataSources {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DatasourceDataType,
RefID: ds.UID,
Name: ds.Name,
Data: ds,
})
}
for _, dashboard := range dashboards {
dashboard.Data.Del("id")
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DashboardDataType,
RefID: dashboard.UID,
Name: dashboard.Title,
Data: map[string]any{"dashboard": dashboard.Data},
})
}
for _, f := range folders {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.FolderDataType,
RefID: f.UID,
Name: f.Title,
Data: f,
})
}
migrationData := &cloudmigration.MigrateDataRequest{
Items: migrationDataSlice,
}
return migrationData, nil
}
func (s *Service) getDataSources(ctx context.Context) ([]datasources.AddDataSourceCommand, error) {
dataSources, err := s.dsService.GetAllDataSources(ctx, &datasources.GetAllDataSourcesQuery{})
if err != nil {
s.log.Error("Failed to get all datasources", "err", err)
return nil, err
}
result := []datasources.AddDataSourceCommand{}
for _, dataSource := range dataSources {
// Decrypt secure json to send raw credentials
decryptedData, err := s.secretsService.DecryptJsonData(ctx, dataSource.SecureJsonData)
if err != nil {
s.log.Error("Failed to decrypt secure json data", "err", err)
return nil, err
}
dataSourceCmd := datasources.AddDataSourceCommand{
OrgID: dataSource.OrgID,
Name: dataSource.Name,
Type: dataSource.Type,
Access: dataSource.Access,
URL: dataSource.URL,
User: dataSource.User,
Database: dataSource.Database,
BasicAuth: dataSource.BasicAuth,
BasicAuthUser: dataSource.BasicAuthUser,
WithCredentials: dataSource.WithCredentials,
IsDefault: dataSource.IsDefault,
JsonData: dataSource.JsonData,
SecureJsonData: decryptedData,
ReadOnly: dataSource.ReadOnly,
UID: dataSource.UID,
}
result = append(result, dataSourceCmd)
}
return result, err
}
func (s *Service) getFolders(ctx context.Context) ([]folder.Folder, error) {
reqCtx := contexthandler.FromContext(ctx)
folders, err := s.folderService.GetFolders(ctx, folder.GetFoldersQuery{
SignedInUser: reqCtx.SignedInUser,
})
if err != nil {
return nil, err
}
result := make([]folder.Folder, len(folders))
for i, folder := range folders {
result[i] = *folder
}
return result, nil
}
func (s *Service) getDashboards(ctx context.Context) ([]dashboards.Dashboard, error) {
dashs, err := s.dashboardService.GetAllDashboards(ctx)
if err != nil {
return nil, err
}
result := make([]dashboards.Dashboard, len(dashs))
for i, dashboard := range dashs {
result[i] = *dashboard
}
return result, nil
}
// asynchronous process for writing the snapshot to the filesystem and updating the snapshot status
func (s *Service) buildSnapshot(ctx context.Context, snapshotMeta cloudmigration.CloudMigrationSnapshot) {
// TODO -- make sure we can only build one snapshot at a time
s.buildSnapshotMutex.Lock()
defer s.buildSnapshotMutex.Unlock()
s.buildSnapshotError = false
// update snapshot status to creating, add some retries since this is a background task
if err := retryer.Retry(func() (retryer.RetrySignal, error) {
err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotMeta.UID,
Status: cloudmigration.SnapshotStatusCreating,
})
return retryer.FuncComplete, err
}, 10, time.Millisecond*100, time.Second*10); err != nil {
s.log.Error("failed to set snapshot status to 'creating'", "err", err)
s.buildSnapshotError = true
return
}
// build snapshot
// just sleep for now to simulate snapshot creation happening
// need to do a couple of fancy things when we implement this:
// - some sort of regular check-in so we know we haven't timed out
// - a channel to listen for cancel events
// - retries baked into the snapshot writing process?
s.log.Debug("snapshot meta", "snapshot", snapshotMeta)
time.Sleep(3 * time.Second)
// update snapshot status to pending upload with retry
if err := retryer.Retry(func() (retryer.RetrySignal, error) {
err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotMeta.UID,
Status: cloudmigration.SnapshotStatusPendingUpload,
})
return retryer.FuncComplete, err
}, 10, time.Millisecond*100, time.Second*10); err != nil {
s.log.Error("failed to set snapshot status to 'pending upload'", "err", err)
s.buildSnapshotError = true
}
}
// asynchronous process for and updating the snapshot status
func (s *Service) uploadSnapshot(ctx context.Context, snapshotMeta cloudmigration.CloudMigrationSnapshot) {
// TODO -- make sure we can only upload one snapshot at a time
s.buildSnapshotMutex.Lock()
defer s.buildSnapshotMutex.Unlock()
s.buildSnapshotError = false
// update snapshot status to uploading, add some retries since this is a background task
if err := retryer.Retry(func() (retryer.RetrySignal, error) {
err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotMeta.UID,
Status: cloudmigration.SnapshotStatusUploading,
})
return retryer.FuncComplete, err
}, 10, time.Millisecond*100, time.Second*10); err != nil {
s.log.Error("failed to set snapshot status to 'creating'", "err", err)
s.buildSnapshotError = true
return
}
// upload snapshot
// just sleep for now to simulate snapshot creation happening
s.log.Debug("snapshot meta", "snapshot", snapshotMeta)
time.Sleep(3 * time.Second)
// update snapshot status to pending processing with retry
if err := retryer.Retry(func() (retryer.RetrySignal, error) {
err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotMeta.UID,
Status: cloudmigration.SnapshotStatusPendingProcessing,
})
return retryer.FuncComplete, err
}, 10, time.Millisecond*100, time.Second*10); err != nil {
s.log.Error("failed to set snapshot status to 'pending upload'", "err", err)
s.buildSnapshotError = true
}
// simulate the rest
// processing
time.Sleep(3 * time.Second)
if err := s.store.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotMeta.UID,
Status: cloudmigration.SnapshotStatusProcessing,
}); err != nil {
s.log.Error("updating snapshot", "err", err)
}
// end here as the GetSnapshot handler will fill in the rest when called
}
pkg/services/cloudmigration/cloudmigrationimpl/store.go
+5
View File
@@ -15,4 +15,9 @@ type store interface {
CreateMigrationRun(ctx context.Context, cmr cloudmigration.CloudMigrationSnapshot) (string, error)
GetMigrationStatus(ctx context.Context, cmrUID string) (*cloudmigration.CloudMigrationSnapshot, error)
GetMigrationStatusList(ctx context.Context, migrationUID string) ([]*cloudmigration.CloudMigrationSnapshot, error)
CreateSnapshot(ctx context.Context, snapshot cloudmigration.CloudMigrationSnapshot) (string, error)
UpdateSnapshot(ctx context.Context, snapshot cloudmigration.UpdateSnapshotCmd) error
GetSnapshotByUID(ctx context.Context, uid string) (*cloudmigration.CloudMigrationSnapshot, error)
GetSnapshotList(ctx context.Context, query cloudmigration.ListSnapshotsQuery) ([]cloudmigration.CloudMigrationSnapshot, error)
}
pkg/services/cloudmigration/cloudmigrationimpl/xorm_store.go
+126
View File
@@ -146,6 +146,106 @@ func (ss *sqlStore) GetMigrationStatusList(ctx context.Context, migrationUID str
return runs, nil
}
func (ss *sqlStore) CreateSnapshot(ctx context.Context, snapshot cloudmigration.CloudMigrationSnapshot) (string, error) {
if err := ss.encryptKey(ctx, &snapshot); err != nil {
return "", err
}
if snapshot.Result == nil {
snapshot.Result = make([]byte, 0)
}
if snapshot.UID == "" {
snapshot.UID = util.GenerateShortUID()
}
err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
snapshot.Created = time.Now()
snapshot.Updated = time.Now()
snapshot.UID = util.GenerateShortUID()
_, err := sess.Insert(&snapshot)
if err != nil {
return err
}
return nil
})
if err != nil {
return "", err
}
return snapshot.UID, nil
}
// UpdateSnapshot takes a snapshot object containing a uid and updates a subset of features in the database.
func (ss *sqlStore) UpdateSnapshot(ctx context.Context, update cloudmigration.UpdateSnapshotCmd) error {
if update.UID == "" {
return fmt.Errorf("missing snapshot uid")
}
err := ss.db.InTransaction(ctx, func(ctx context.Context) error {
// Update status if set
if err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
if update.Status != "" {
rawSQL := "UPDATE cloud_migration_snapshot SET status=? WHERE uid=?"
if _, err := sess.Exec(rawSQL, update.Status, update.UID); err != nil {
return fmt.Errorf("updating snapshot status for uid %s: %w", update.UID, err)
}
}
return nil
}); err != nil {
return err
}
// Update result if set
if err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
if len(update.Result) > 0 {
rawSQL := "UPDATE cloud_migration_snapshot SET result=? WHERE uid=?"
if _, err := sess.Exec(rawSQL, update.Result, update.UID); err != nil {
return fmt.Errorf("updating snapshot result for uid %s: %w", update.UID, err)
}
}
return nil
}); err != nil {
return err
}
return nil
})
return err
}
func (ss *sqlStore) GetSnapshotByUID(ctx context.Context, uid string) (*cloudmigration.CloudMigrationSnapshot, error) {
var snapshot cloudmigration.CloudMigrationSnapshot
err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
exist, err := sess.Where("uid=?", uid).Get(&snapshot)
if err != nil {
return err
}
if !exist {
return cloudmigration.ErrSnapshotNotFound
}
return nil
})
if err := ss.decryptKey(ctx, &snapshot); err != nil {
return &snapshot, err
}
return &snapshot, err
}
func (ss *sqlStore) GetSnapshotList(ctx context.Context, query cloudmigration.ListSnapshotsQuery) ([]cloudmigration.CloudMigrationSnapshot, error) {
var runs = make([]cloudmigration.CloudMigrationSnapshot, 0)
err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
sess.Limit(query.Limit, query.Offset)
return sess.Find(&runs, &cloudmigration.CloudMigrationSnapshot{
SessionUID: query.SessionUID,
})
})
if err != nil {
return nil, err
}
return runs, nil
}
func (ss *sqlStore) encryptToken(ctx context.Context, cm *cloudmigration.CloudMigrationSession) error {
s, err := ss.secretsService.Encrypt(ctx, []byte(cm.AuthToken), secrets.WithoutScope())
if err != nil {
@@ -171,3 +271,29 @@ func (ss *sqlStore) decryptToken(ctx context.Context, cm *cloudmigration.CloudMi
return nil
}
func (ss *sqlStore) encryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
s, err := ss.secretsService.Encrypt(ctx, []byte(snapshot.EncryptionKey), secrets.WithoutScope())
if err != nil {
return fmt.Errorf("encrypting key: %w", err)
}
snapshot.EncryptionKey = base64.StdEncoding.EncodeToString(s)
return nil
}
func (ss *sqlStore) decryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
decoded, err := base64.StdEncoding.DecodeString(snapshot.EncryptionKey)
if err != nil {
return fmt.Errorf("key could not be decoded")
}
t, err := ss.secretsService.Decrypt(ctx, decoded)
if err != nil {
return fmt.Errorf("decrypting key: %w", err)
}
snapshot.EncryptionKey = string(t)
return nil
}
pkg/services/cloudmigration/cloudmigrationimpl/xorm_store_test.go
+4 -5
View File
@@ -152,7 +152,6 @@ func Test_GetMigrationStatusList(t *testing.T) {
list, err := s.GetMigrationStatusList(ctx, "qwerty")
require.NoError(t, err)
require.Equal(t, 2, len(list))
// TODO validate that this is ok
})
t.Run("returns no error if migration was not found, just empty list", func(t *testing.T) {
@@ -188,11 +187,11 @@ func setUpTest(t *testing.T) (*sqlstore.SQLStore, *sqlStore) {
// insert cloud migration run test data
_, err = testDB.GetSqlxSession().Exec(ctx, `
INSERT INTO
cloud_migration_snapshot (session_uid, uid, result, created, updated, finished)
cloud_migration_snapshot (session_uid, uid, result, created, updated, finished, status)
VALUES
('qwerty', 'poiuy', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000'),
('qwerty', 'lkjhg', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000'),
('zxcvbn', 'mnbvvc', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000');
('qwerty', 'poiuy', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000', "finished"),
('qwerty', 'lkjhg', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000', "finished"),
('zxcvbn', 'mnbvvc', ?, '2024-03-25 15:30:36.000', '2024-03-27 15:30:43.000', '2024-03-27 15:30:43.000', "finished");
`,
[]byte("ERROR"),
[]byte("OK"),
pkg/services/cloudmigration/gmsclient/client.go
+2
View File
@@ -9,6 +9,8 @@ import (
type Client interface {
ValidateKey(context.Context, cloudmigration.CloudMigrationSession) error
MigrateData(context.Context, cloudmigration.CloudMigrationSession, cloudmigration.MigrateDataRequest) (*cloudmigration.MigrateDataResponse, error)
InitializeSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.InitializeSnapshotResponse, error)
GetSnapshotStatus(context.Context, cloudmigration.CloudMigrationSession, cloudmigration.CloudMigrationSnapshot) (*cloudmigration.CloudMigrationSnapshot, error)
}
const logPrefix = "cloudmigration.gmsclient"
pkg/services/cloudmigration/gmsclient/gms_client.go
+8
View File
@@ -111,6 +111,14 @@ func (c *gmsClientImpl) MigrateData(ctx context.Context, cm cloudmigration.Cloud
return &result, nil
}
func (c *gmsClientImpl) InitializeSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.InitializeSnapshotResponse, error) {
panic("not implemented")
}
func (c *gmsClientImpl) GetSnapshotStatus(context.Context, cloudmigration.CloudMigrationSession, cloudmigration.CloudMigrationSnapshot) (*cloudmigration.CloudMigrationSnapshot, error) {
panic("not implemented")
}
func convertRequestToDTO(request cloudmigration.MigrateDataRequest) MigrateDataRequestDTO {
items := make([]MigrateDataRequestItemDTO, len(request.Items))
for i := 0; i < len(request.Items); i++ {
pkg/services/cloudmigration/gmsclient/inmemory_client.go
+53 -1
View File
@@ -2,9 +2,12 @@ package gmsclient
import (
"context"
"encoding/json"
"math/rand"
"time"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/util"
)
// NewInMemoryClient returns an implementation of Client that returns canned responses
@@ -12,7 +15,9 @@ func NewInMemoryClient() Client {
return &memoryClientImpl{}
}
type memoryClientImpl struct{}
type memoryClientImpl struct {
snapshot *cloudmigration.InitializeSnapshotResponse
}
func (c *memoryClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigrationSession) error {
return nil
@@ -43,3 +48,50 @@ func (c *memoryClientImpl) MigrateData(
return &result, nil
}
func (c *memoryClientImpl) InitializeSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.InitializeSnapshotResponse, error) {
c.snapshot = &cloudmigration.InitializeSnapshotResponse{
EncryptionKey: util.GenerateShortUID(),
GMSSnapshotUID: util.GenerateShortUID(),
UploadURL: "localhost:3000",
}
return c.snapshot, nil
}
func (c *memoryClientImpl) GetSnapshotStatus(ctx context.Context, session cloudmigration.CloudMigrationSession, snapshot cloudmigration.CloudMigrationSnapshot) (*cloudmigration.CloudMigrationSnapshot, error) {
// just fake an entire response
gmsSnapshot := cloudmigration.CloudMigrationSnapshot{
Status: cloudmigration.SnapshotStatusFinished,
GMSSnapshotUID: util.GenerateShortUID(),
Result: []byte{},
Finished: time.Now(),
}
result := []cloudmigration.MigrateDataResponseItem{
{
Type: cloudmigration.DashboardDataType,
RefID: util.GenerateShortUID(),
Status: cloudmigration.ItemStatusOK,
},
{
Type: cloudmigration.DatasourceDataType,
RefID: util.GenerateShortUID(),
Status: cloudmigration.ItemStatusError,
Error: "fake error",
},
{
Type: cloudmigration.FolderDataType,
RefID: util.GenerateShortUID(),
Status: cloudmigration.ItemStatusOK,
},
}
b, err := json.Marshal(result)
if err != nil {
return nil, err
}
gmsSnapshot.Result = b
return &gmsSnapshot, nil
}
pkg/services/cloudmigration/model.go
+72 -11
View File
@@ -15,6 +15,7 @@ var (
ErrMigrationRunNotFound = errutil.NotFound("cloudmigrations.migrationRunNotFound").Errorf("Migration run not found")
ErrMigrationNotDeleted = errutil.Internal("cloudmigrations.sessionNotDeleted").Errorf("Session not deleted")
ErrTokenNotFound = errutil.NotFound("cloudmigrations.tokenNotFound").Errorf("Token not found")
ErrSnapshotNotFound = errutil.NotFound("cloudmigrations.snapshotNotFound").Errorf("Snapshot not found")
)
// CloudMigration domain structs
@@ -31,26 +32,64 @@ type CloudMigrationSession struct {
}
type CloudMigrationSnapshot struct {
ID int64 `xorm:"pk autoincr 'id'"`
UID string `xorm:"uid"`
SessionUID string `xorm:"session_uid"`
Result []byte //store raw gms response body
Created time.Time
Updated time.Time
Finished time.Time
ID int64 `xorm:"pk autoincr 'id'"`
UID string `xorm:"uid"`
SessionUID string `xorm:"session_uid"`
Status SnapshotStatus
EncryptionKey string `xorm:"encryption_key"` // stored in the unified secrets table
UploadURL string `xorm:"upload_url"`
LocalDir string `xorm:"local_directory"`
GMSSnapshotUID string `xorm:"gms_snapshot_uid"`
ErrorString string `xorm:"error_string"`
Created time.Time
Updated time.Time
Finished time.Time
// []MigrateDataResponseItem
Result []byte `xorm:"result"` //store raw gms response body
}
func (r CloudMigrationSnapshot) GetResult() (*MigrateDataResponse, error) {
type SnapshotStatus string
const (
SnapshotStatusInitializing = "initializing"
SnapshotStatusCreating = "creating"
SnapshotStatusPendingUpload = "pending_upload"
SnapshotStatusUploading = "uploading"
SnapshotStatusPendingProcessing = "pending_processing"
SnapshotStatusProcessing = "processing"
SnapshotStatusFinished = "finished"
SnapshotStatusError = "error"
SnapshotStatusUnknown = "unknown"
)
// Deprecated, use GetSnapshotResult for the async workflow
func (s CloudMigrationSnapshot) GetResult() (*MigrateDataResponse, error) {
var result MigrateDataResponse
err := json.Unmarshal(r.Result, &result)
err := json.Unmarshal(s.Result, &result)
if err != nil {
return nil, errors.New("could not parse result of run")
}
result.RunUID = r.UID
result.RunUID = s.UID
return &result, nil
}
type SnapshotList struct {
func (s CloudMigrationSnapshot) ShouldQueryGMS() bool {
return s.Status == SnapshotStatusPendingProcessing || s.Status == SnapshotStatusProcessing
}
func (s CloudMigrationSnapshot) GetSnapshotResult() ([]MigrateDataResponseItem, error) {
var result []MigrateDataResponseItem
if len(s.Result) > 0 {
err := json.Unmarshal(s.Result, &result)
if err != nil {
return nil, errors.New("could not parse result of run")
}
}
return result, nil
}
type CloudMigrationRunList struct {
Runs []MigrateDataResponseList
}
@@ -69,6 +108,18 @@ type CloudMigrationSessionListResponse struct {
Sessions []CloudMigrationSessionResponse
}
type ListSnapshotsQuery struct {
SessionUID string
Offset int
Limit int
}
type UpdateSnapshotCmd struct {
UID string
Status SnapshotStatus
Result []byte //store raw gms response body
}
// access token
type CreateAccessTokenResponse struct {
@@ -140,3 +191,13 @@ type MigrateDataResponseItem struct {
Status ItemStatus
Error string
}
type CreateSessionResponse struct {
SnapshotUid string
}
type InitializeSnapshotResponse struct {
EncryptionKey string
UploadURL string
GMSSnapshotUID string
}
pkg/services/featuremgmt/registry.go
+1 -1
View File
@@ -709,7 +709,7 @@ var (
AllowSelfServe: false,
RequiresRestart: true,
},
FeatureFlag{
{
Name: "disableClassicHTTPHistogram",
Description: "Disables classic HTTP Histogram (use with enableNativeHTTPHistogram)",
Stage: FeatureStageExperimental,
pkg/services/sqlstore/migrations/cloud_migrations.go
+21 -2
View File
@@ -5,7 +5,7 @@ import (
)
func addCloudMigrationsMigrations(mg *Migrator) {
// v1 - synchronous workflow
// --- v1 - synchronous workflow
migrationTable := Table{
Name: "cloud_migration",
Columns: []*Column{
@@ -65,7 +65,7 @@ func addCloudMigrationsMigrations(mg *Migrator) {
Cols: []string{"uid"}, Type: UniqueIndex,
}))
// v2 - asynchronous workflow refactor
// --- v2 - asynchronous workflow refactor
sessionTable := Table{
Name: "cloud_migration_session",
Columns: []*Column{
@@ -120,4 +120,23 @@ func addCloudMigrationsMigrations(mg *Migrator) {
"updated": "updated",
"finished": "finished",
})
// --- add new columns to snapshots table
uploadUrlColumn := Column{Name: "upload_url", Type: DB_Text, Nullable: true}
mg.AddMigration("add snapshot upload_url column", NewAddColumnMigration(migrationSnapshotTable, &uploadUrlColumn))
statusColumn := Column{Name: "status", Type: DB_Text, Nullable: false}
mg.AddMigration("add snapshot status column", NewAddColumnMigration(migrationSnapshotTable, &statusColumn))
localDirColumn := Column{Name: "local_directory", Type: DB_Text, Nullable: true}
mg.AddMigration("add snapshot local_directory column", NewAddColumnMigration(migrationSnapshotTable, &localDirColumn))
gmsSnapshotUIDColumn := Column{Name: "gms_snapshot_uid", Type: DB_Text, Nullable: true}
mg.AddMigration("add snapshot gms_snapshot_uid column", NewAddColumnMigration(migrationSnapshotTable, &gmsSnapshotUIDColumn))
encryptionKeyColumn := Column{Name: "encryption_key", Type: DB_Text, Nullable: true}
mg.AddMigration("add snapshot encryption_key column", NewAddColumnMigration(migrationSnapshotTable, &encryptionKeyColumn))
errorStringColumn := Column{Name: "error_string", Type: DB_Text, Nullable: true}
mg.AddMigration("add snapshot error_string column", NewAddColumnMigration(migrationSnapshotTable, &errorStringColumn))
}
public/api-enterprise-spec.json
+114 -4
View File
@@ -3159,6 +3159,17 @@
}
}
},
"CloudMigrationRunListDTO": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseListDTO"
}
}
}
},
"CloudMigrationSessionListResponseDTO": {
"type": "object",
"properties": {
@@ -3632,6 +3643,14 @@
}
}
},
"CreateSnapshotResponseDTO": {
"type": "object",
"properties": {
"uid": {
"type": "string"
}
}
},
"CreateTeamCommand": {
"type": "object",
"properties": {
@@ -4726,6 +4745,45 @@
}
]
},
"GetSnapshotResponseDTO": {
"type": "object",
"properties": {
"created": {
"type": "string",
"format": "date-time"
},
"finished": {
"type": "string",
"format": "date-time"
},
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseItemDTO"
}
},
"sessionUid": {
"type": "string"
},
"status": {
"type": "string",
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
]
},
"uid": {
"type": "string"
}
}
},
"Hit": {
"type": "object",
"properties": {
@@ -7084,13 +7142,47 @@
"type": "integer",
"format": "int64"
},
"SnapshotListDTO": {
"SnapshotDTO": {
"description": "Base snapshot without results",
"type": "object",
"properties": {
"runs": {
"created": {
"type": "string",
"format": "date-time"
},
"finished": {
"type": "string",
"format": "date-time"
},
"sessionUid": {
"type": "string"
},
"status": {
"type": "string",
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
]
},
"uid": {
"type": "string"
}
}
},
"SnapshotListResponseDTO": {
"type": "object",
"properties": {
"snapshots": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseListDTO"
"$ref": "#/definitions/SnapshotDTO"
}
}
}
@@ -8394,7 +8486,7 @@
"cloudMigrationRunListResponse": {
"description": "",
"schema": {
"$ref": "#/definitions/SnapshotListDTO"
"$ref": "#/definitions/CloudMigrationRunListDTO"
}
},
"cloudMigrationRunResponse": {
@@ -8559,6 +8651,12 @@
"$ref": "#/definitions/ServiceAccountDTO"
}
},
"createSnapshotResponse": {
"description": "",
"schema": {
"$ref": "#/definitions/CreateSnapshotResponseDTO"
}
},
"createTeamResponse": {
"description": "",
"schema": {
@@ -9126,6 +9224,12 @@
}
}
},
"getSnapshotResponse": {
"description": "",
"schema": {
"$ref": "#/definitions/GetSnapshotResponseDTO"
}
},
"getStatusResponse": {
"description": ""
},
@@ -9582,6 +9686,12 @@
"$ref": "#/definitions/RoleAssignmentsDTO"
}
},
"snapshotListResponse": {
"description": "",
"schema": {
"$ref": "#/definitions/SnapshotListResponseDTO"
}
},
"unauthorisedError": {
"description": "UnauthorizedError is returned when the request is not authenticated.",
"schema": {
public/api-merged.json
+349 -4
View File
@@ -2336,6 +2336,9 @@
"200": {
"$ref": "#/responses/cloudMigrationSessionResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2368,6 +2371,9 @@
"200": {
"$ref": "#/responses/cloudMigrationRunResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2400,6 +2406,9 @@
"200": {
"$ref": "#/responses/cloudMigrationSessionResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2427,6 +2436,9 @@
}
],
"responses": {
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2459,6 +2471,9 @@
"200": {
"$ref": "#/responses/cloudMigrationRunListResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2490,6 +2505,223 @@
"200": {
"$ref": "#/responses/cloudMigrationRunResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
"403": {
"$ref": "#/responses/forbiddenError"
},
"500": {
"$ref": "#/responses/internalServerError"
}
}
}
},
"/cloudmigration/migration/{uid}/snapshot": {
"post": {
"description": "If the snapshot initialization is successful, the snapshot uid is returned.",
"tags": [
"migrations"
],
"summary": "Trigger the creation of an instance snapshot associated with the provided session.",
"operationId": "createSnapshot",
"parameters": [
{
"type": "string",
"description": "UID of a session",
"name": "uid",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"$ref": "#/responses/createSnapshotResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
"403": {
"$ref": "#/responses/forbiddenError"
},
"500": {
"$ref": "#/responses/internalServerError"
}
}
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}": {
"get": {
"tags": [
"migrations"
],
"summary": "Get metadata about a snapshot, including where it is in its processing and final results.",
"operationId": "getSnapshot",
"parameters": [
{
"type": "string",
"description": "Session UID of a session",
"name": "uid",
"in": "path",
"required": true
},
{
"type": "string",
"description": "UID of a snapshot",
"name": "snapshotUid",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"$ref": "#/responses/getSnapshotResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
"403": {
"$ref": "#/responses/forbiddenError"
},
"500": {
"$ref": "#/responses/internalServerError"
}
}
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}/cancel": {
"post": {
"description": "TODO: Implement",
"tags": [
"migrations"
],
"summary": "Cancel a snapshot, wherever it is in its processing chain.",
"operationId": "cancelSnapshot",
"parameters": [
{
"type": "string",
"description": "Session UID of a session",
"name": "uid",
"in": "path",
"required": true
},
{
"type": "string",
"description": "UID of a snapshot",
"name": "snapshotUid",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "(empty)"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
"403": {
"$ref": "#/responses/forbiddenError"
},
"500": {
"$ref": "#/responses/internalServerError"
}
}
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}/upload": {
"post": {
"tags": [
"migrations"
],
"summary": "Upload a snapshot to the Grafana Migration Service for processing.",
"operationId": "uploadSnapshot",
"parameters": [
{
"type": "string",
"description": "Session UID of a session",
"name": "uid",
"in": "path",
"required": true
},
{
"type": "string",
"description": "UID of a snapshot",
"name": "snapshotUid",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "(empty)"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
"403": {
"$ref": "#/responses/forbiddenError"
},
"500": {
"$ref": "#/responses/internalServerError"
}
}
}
},
"/cloudmigration/migration/{uid}/snapshots": {
"get": {
"tags": [
"migrations"
],
"summary": "Get a list of snapshots for a session.",
"operationId": "getShapshotList",
"parameters": [
{
"type": "integer",
"format": "int64",
"default": 0,
"description": "Offset is used for pagination with limit",
"name": "offset",
"in": "query"
},
{
"type": "integer",
"format": "int64",
"default": 100,
"description": "Max limit for results returned.",
"name": "limit",
"in": "query"
},
{
"type": "string",
"description": "Session UID of a session",
"name": "uid",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"$ref": "#/responses/snapshotListResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -2569,6 +2801,9 @@
"204": {
"$ref": "#/responses/cloudMigrationDeleteTokenResponse"
},
"400": {
"$ref": "#/responses/badRequestError"
},
"401": {
"$ref": "#/responses/unauthorisedError"
},
@@ -13336,6 +13571,17 @@
}
}
},
"CloudMigrationRunListDTO": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseListDTO"
}
}
}
},
"CloudMigrationSessionListResponseDTO": {
"type": "object",
"properties": {
@@ -13877,6 +14123,14 @@
}
}
},
"CreateSnapshotResponseDTO": {
"type": "object",
"properties": {
"uid": {
"type": "string"
}
}
},
"CreateTeamCommand": {
"type": "object",
"properties": {
@@ -15270,6 +15524,45 @@
}
]
},
"GetSnapshotResponseDTO": {
"type": "object",
"properties": {
"created": {
"type": "string",
"format": "date-time"
},
"finished": {
"type": "string",
"format": "date-time"
},
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseItemDTO"
}
},
"sessionUid": {
"type": "string"
},
"status": {
"type": "string",
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
]
},
"uid": {
"type": "string"
}
}
},
"GettableAlertmanagers": {
"type": "object",
"properties": {
@@ -19985,13 +20278,47 @@
"SmtpNotEnabled": {
"$ref": "#/definitions/ResponseDetails"
},
"SnapshotListDTO": {
"SnapshotDTO": {
"description": "Base snapshot without results",
"type": "object",
"properties": {
"runs": {
"created": {
"type": "string",
"format": "date-time"
},
"finished": {
"type": "string",
"format": "date-time"
},
"sessionUid": {
"type": "string"
},
"status": {
"type": "string",
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
]
},
"uid": {
"type": "string"
}
}
},
"SnapshotListResponseDTO": {
"type": "object",
"properties": {
"snapshots": {
"type": "array",
"items": {
"$ref": "#/definitions/MigrateDataResponseListDTO"
"$ref": "#/definitions/SnapshotDTO"
}
}
}
@@ -22434,7 +22761,7 @@
"cloudMigrationRunListResponse": {
"description": "(empty)",
"schema": {
"$ref": "#/definitions/SnapshotListDTO"
"$ref": "#/definitions/CloudMigrationRunListDTO"
}
},
"cloudMigrationRunResponse": {
@@ -22599,6 +22926,12 @@
"$ref": "#/definitions/ServiceAccountDTO"
}
},
"createSnapshotResponse": {
"description": "(empty)",
"schema": {
"$ref": "#/definitions/CreateSnapshotResponseDTO"
}
},
"createTeamResponse": {
"description": "(empty)",
"schema": {
@@ -23166,6 +23499,12 @@
}
}
},
"getSnapshotResponse": {
"description": "(empty)",
"schema": {
"$ref": "#/definitions/GetSnapshotResponseDTO"
}
},
"getStatusResponse": {
"description": "(empty)"
},
@@ -23631,6 +23970,12 @@
"$ref": "#/definitions/RoleAssignmentsDTO"
}
},
"snapshotListResponse": {
"description": "(empty)",
"schema": {
"$ref": "#/definitions/SnapshotListResponseDTO"
}
},
"unauthorisedError": {
"description": "UnauthorizedError is returned when the request is not authenticated.",
"schema": {
public/openapi3.json
+381 -4
View File
@@ -200,7 +200,7 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SnapshotListDTO"
"$ref": "#/components/schemas/CloudMigrationRunListDTO"
}
}
},
@@ -424,6 +424,16 @@
},
"description": "(empty)"
},
"createSnapshotResponse": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateSnapshotResponseDTO"
}
}
},
"description": "(empty)"
},
"createTeamResponse": {
"content": {
"application/json": {
@@ -1251,6 +1261,16 @@
},
"description": "(empty)"
},
"getSnapshotResponse": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetSnapshotResponseDTO"
}
}
},
"description": "(empty)"
},
"getStatusResponse": {
"description": "(empty)"
},
@@ -1912,6 +1932,16 @@
},
"description": "(empty)"
},
"snapshotListResponse": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SnapshotListResponseDTO"
}
}
},
"description": "(empty)"
},
"unauthorisedError": {
"content": {
"application/json": {
@@ -3683,6 +3713,17 @@
},
"type": "object"
},
"CloudMigrationRunListDTO": {
"properties": {
"runs": {
"items": {
"$ref": "#/components/schemas/MigrateDataResponseListDTO"
},
"type": "array"
}
},
"type": "object"
},
"CloudMigrationSessionListResponseDTO": {
"properties": {
"sessions": {
@@ -4224,6 +4265,14 @@
},
"type": "object"
},
"CreateSnapshotResponseDTO": {
"properties": {
"uid": {
"type": "string"
}
},
"type": "object"
},
"CreateTeamCommand": {
"properties": {
"email": {
@@ -5617,6 +5666,45 @@
],
"title": "Get home dashboard response."
},
"GetSnapshotResponseDTO": {
"properties": {
"created": {
"format": "date-time",
"type": "string"
},
"finished": {
"format": "date-time",
"type": "string"
},
"results": {
"items": {
"$ref": "#/components/schemas/MigrateDataResponseItemDTO"
},
"type": "array"
},
"sessionUid": {
"type": "string"
},
"status": {
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
],
"type": "string"
},
"uid": {
"type": "string"
}
},
"type": "object"
},
"GettableAlertmanagers": {
"properties": {
"data": {
@@ -10331,11 +10419,45 @@
"SmtpNotEnabled": {
"$ref": "#/components/schemas/ResponseDetails"
},
"SnapshotListDTO": {
"SnapshotDTO": {
"description": "Base snapshot without results",
"properties": {
"runs": {
"created": {
"format": "date-time",
"type": "string"
},
"finished": {
"format": "date-time",
"type": "string"
},
"sessionUid": {
"type": "string"
},
"status": {
"enum": [
"INITIALIZING",
"CREATING",
"PENDING_UPLOAD",
"UPLOADING",
"PENDING_PROCESSING",
"PROCESSING",
"FINISHED",
"ERROR",
"UNKNOWN"
],
"type": "string"
},
"uid": {
"type": "string"
}
},
"type": "object"
},
"SnapshotListResponseDTO": {
"properties": {
"snapshots": {
"items": {
"$ref": "#/components/schemas/MigrateDataResponseListDTO"
"$ref": "#/components/schemas/SnapshotDTO"
},
"type": "array"
}
@@ -15165,6 +15287,9 @@
"200": {
"$ref": "#/components/responses/cloudMigrationSessionResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15199,6 +15324,9 @@
"200": {
"$ref": "#/components/responses/cloudMigrationRunResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15230,6 +15358,9 @@
}
],
"responses": {
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15262,6 +15393,9 @@
"200": {
"$ref": "#/components/responses/cloudMigrationSessionResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15296,6 +15430,9 @@
"200": {
"$ref": "#/components/responses/cloudMigrationRunListResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15329,6 +15466,9 @@
"200": {
"$ref": "#/components/responses/cloudMigrationRunResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
@@ -15345,6 +15485,240 @@
]
}
},
"/cloudmigration/migration/{uid}/snapshot": {
"post": {
"description": "If the snapshot initialization is successful, the snapshot uid is returned.",
"operationId": "createSnapshot",
"parameters": [
{
"description": "UID of a session",
"in": "path",
"name": "uid",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"$ref": "#/components/responses/createSnapshotResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
"403": {
"$ref": "#/components/responses/forbiddenError"
},
"500": {
"$ref": "#/components/responses/internalServerError"
}
},
"summary": "Trigger the creation of an instance snapshot associated with the provided session.",
"tags": [
"migrations"
]
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}": {
"get": {
"operationId": "getSnapshot",
"parameters": [
{
"description": "Session UID of a session",
"in": "path",
"name": "uid",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "UID of a snapshot",
"in": "path",
"name": "snapshotUid",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"$ref": "#/components/responses/getSnapshotResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
"403": {
"$ref": "#/components/responses/forbiddenError"
},
"500": {
"$ref": "#/components/responses/internalServerError"
}
},
"summary": "Get metadata about a snapshot, including where it is in its processing and final results.",
"tags": [
"migrations"
]
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}/cancel": {
"post": {
"description": "TODO: Implement",
"operationId": "cancelSnapshot",
"parameters": [
{
"description": "Session UID of a session",
"in": "path",
"name": "uid",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "UID of a snapshot",
"in": "path",
"name": "snapshotUid",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "(empty)"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
"403": {
"$ref": "#/components/responses/forbiddenError"
},
"500": {
"$ref": "#/components/responses/internalServerError"
}
},
"summary": "Cancel a snapshot, wherever it is in its processing chain.",
"tags": [
"migrations"
]
}
},
"/cloudmigration/migration/{uid}/snapshot/{snapshotUid}/upload": {
"post": {
"operationId": "uploadSnapshot",
"parameters": [
{
"description": "Session UID of a session",
"in": "path",
"name": "uid",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "UID of a snapshot",
"in": "path",
"name": "snapshotUid",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "(empty)"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
"403": {
"$ref": "#/components/responses/forbiddenError"
},
"500": {
"$ref": "#/components/responses/internalServerError"
}
},
"summary": "Upload a snapshot to the Grafana Migration Service for processing.",
"tags": [
"migrations"
]
}
},
"/cloudmigration/migration/{uid}/snapshots": {
"get": {
"operationId": "getShapshotList",
"parameters": [
{
"description": "Offset is used for pagination with limit",
"in": "query",
"name": "offset",
"schema": {
"default": 0,
"format": "int64",
"type": "integer"
}
},
{
"description": "Max limit for results returned.",
"in": "query",
"name": "limit",
"schema": {
"default": 100,
"format": "int64",
"type": "integer"
}
},
{
"description": "Session UID of a session",
"in": "path",
"name": "uid",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"$ref": "#/components/responses/snapshotListResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},
"403": {
"$ref": "#/components/responses/forbiddenError"
},
"500": {
"$ref": "#/components/responses/internalServerError"
}
},
"summary": "Get a list of snapshots for a session.",
"tags": [
"migrations"
]
}
},
"/cloudmigration/token": {
"get": {
"operationId": "getCloudMigrationToken",
@@ -15410,6 +15784,9 @@
"204": {
"$ref": "#/components/responses/cloudMigrationDeleteTokenResponse"
},
"400": {
"$ref": "#/components/responses/badRequestError"
},
"401": {
"$ref": "#/components/responses/unauthorisedError"
},