CloudMigrations: Implement snapshot management apis (#89296)

* add new apis

* add payloads

* create snapshot status type

* add some impl

* finish implementing update

* start implementing build snapshot func

* add more fake build logic

* add cancel endpoint. do some cleanup

* implement GetSnapshot

* implement upload snapshot

* merge onprem status with gms result

* get it working

* update comment

* rename list endpoint

* add query limit and offset

* add helper method to snapshot

* little bit of cleanup

* work on swagger annotations

* manual merge

* generate swagger specs

* clean up curl commands

* fix bugs found during final testing

* fix linter issue

* fix unit test

pkg/services/cloudmigration/api/api.go

@@ -40,19 +40,28 @@ func RegisterApi(
 // registerEndpoints Registers Endpoints on Grafana Router
 func (cma *CloudMigrationAPI) registerEndpoints() {
 	cma.routeRegister.Group("/api/cloudmigration", func(cloudMigrationRoute routing.RouteRegister) {
+		// destination instance endpoints for token management
+		cloudMigrationRoute.Get("/token", routing.Wrap(cma.GetToken))
+		cloudMigrationRoute.Post("/token", routing.Wrap(cma.CreateToken))
+		cloudMigrationRoute.Delete("/token/:uid", routing.Wrap(cma.DeleteToken))
+
+		// on-prem instance endpoints for managing GMS sessions
 		cloudMigrationRoute.Get("/migration", routing.Wrap(cma.GetSessionList))
 		cloudMigrationRoute.Post("/migration", routing.Wrap(cma.CreateSession))
 		cloudMigrationRoute.Get("/migration/:uid", routing.Wrap(cma.GetSession))
 		cloudMigrationRoute.Delete("/migration/:uid", routing.Wrap(cma.DeleteSession))
 
-		// TODO new APIs for snapshot management to replace these
+		// sync approach to data migration
 		cloudMigrationRoute.Post("/migration/:uid/run", routing.Wrap(cma.RunMigration))
 		cloudMigrationRoute.Get("/migration/:uid/run", routing.Wrap(cma.GetMigrationRunList))
 		cloudMigrationRoute.Get("/migration/run/:runUID", routing.Wrap(cma.GetMigrationRun))
 
-		cloudMigrationRoute.Get("/token", routing.Wrap(cma.GetToken))
-		cloudMigrationRoute.Post("/token", routing.Wrap(cma.CreateToken))
-		cloudMigrationRoute.Delete("/token/:uid", routing.Wrap(cma.DeleteToken))
+		// async approach to data migration using snapshots
+		cloudMigrationRoute.Post("/migration/:uid/snapshot", routing.Wrap(cma.CreateSnapshot))
+		cloudMigrationRoute.Get("/migration/:uid/snapshot/:snapshotUid", routing.Wrap(cma.GetSnapshot))
+		cloudMigrationRoute.Get("/migration/:uid/snapshots", routing.Wrap(cma.GetSnapshotList))
+		cloudMigrationRoute.Post("/migration/:uid/snapshot/:snapshotUid/upload", routing.Wrap(cma.UploadSnapshot))
+		cloudMigrationRoute.Post("/migration/:uid/snapshot/:snapshotUid/cancel", routing.Wrap(cma.CancelSnapshot))
 	}, middleware.ReqOrgAdmin)
 }
 
@@ -121,6 +130,7 @@ func (cma *CloudMigrationAPI) CreateToken(c *contextmodel.ReqContext) response.R
 //
 // Responses:
 // 204: cloudMigrationDeleteTokenResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -170,6 +180,7 @@ func (cma *CloudMigrationAPI) GetSessionList(c *contextmodel.ReqContext) respons
 //
 // Responses:
 // 200: cloudMigrationSessionResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -201,6 +212,7 @@ func (cma *CloudMigrationAPI) GetSession(c *contextmodel.ReqContext) response.Re
 //
 // Responses:
 // 200: cloudMigrationSessionResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -235,6 +247,7 @@ func (cma *CloudMigrationAPI) CreateSession(c *contextmodel.ReqContext) response
 //
 // Responses:
 // 200: cloudMigrationRunResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -261,6 +274,7 @@ func (cma *CloudMigrationAPI) RunMigration(c *contextmodel.ReqContext) response.
 //
 // Responses:
 // 200: cloudMigrationRunResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -293,6 +307,7 @@ func (cma *CloudMigrationAPI) GetMigrationRun(c *contextmodel.ReqContext) respon
 //
 // Responses:
 // 200: cloudMigrationRunListResponse
+// 400: badRequestError
 // 401: unauthorisedError
 // 403: forbiddenError
 // 500: internalServerError
@@ -314,7 +329,7 @@ func (cma *CloudMigrationAPI) GetMigrationRunList(c *contextmodel.ReqContext) re
 	for i := 0; i < len(runList.Runs); i++ {
 		runs[i] = MigrateDataResponseListDTO{runList.Runs[i].RunUID}
 	}
-	return response.JSON(http.StatusOK, SnapshotListDTO{
+	return response.JSON(http.StatusOK, CloudMigrationRunListDTO{
 		Runs: runs,
 	})
 }
@@ -326,6 +341,7 @@ func (cma *CloudMigrationAPI) GetMigrationRunList(c *contextmodel.ReqContext) re
 // Responses:
 // 200
 // 401: unauthorisedError
+// 400: badRequestError
 // 403: forbiddenError
 // 500: internalServerError
 func (cma *CloudMigrationAPI) DeleteSession(c *contextmodel.ReqContext) response.Response {
@@ -343,3 +359,194 @@ func (cma *CloudMigrationAPI) DeleteSession(c *contextmodel.ReqContext) response
 	}
 	return response.Empty(http.StatusOK)
 }
+
+// swagger:route POST /cloudmigration/migration/{uid}/snapshot migrations createSnapshot
+//
+// Trigger the creation of an instance snapshot associated with the provided session.
+// If the snapshot initialization is successful, the snapshot uid is returned.
+//
+// Responses:
+// 200: createSnapshotResponse
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 500: internalServerError
+func (cma *CloudMigrationAPI) CreateSnapshot(c *contextmodel.ReqContext) response.Response {
+	ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.CreateSnapshot")
+	defer span.End()
+
+	uid := web.Params(c.Req)[":uid"]
+	if err := util.ValidateUID(uid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
+	}
+
+	ss, err := cma.cloudMigrationService.CreateSnapshot(ctx, uid)
+	if err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "error creating snapshot", err)
+	}
+
+	return response.JSON(http.StatusOK, CreateSnapshotResponseDTO{
+		SnapshotUID: ss.UID,
+	})
+}
+
+// swagger:route GET /cloudmigration/migration/{uid}/snapshot/{snapshotUid} migrations getSnapshot
+//
+// Get metadata about a snapshot, including where it is in its processing and final results.
+//
+// Responses:
+// 200: getSnapshotResponse
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 500: internalServerError
+func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.Response {
+	ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.GetSnapshot")
+	defer span.End()
+
+	sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
+	if err := util.ValidateUID(sessUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
+	}
+	if err := util.ValidateUID(snapshotUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
+	}
+
+	snapshot, err := cma.cloudMigrationService.GetSnapshot(ctx, sessUid, snapshotUid)
+	if err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "error retrieving snapshot", err)
+	}
+
+	result, err := snapshot.GetSnapshotResult()
+	if err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "error snapshot reading snapshot results", err)
+	}
+
+	dtoResults := make([]MigrateDataResponseItemDTO, len(result))
+	for i := 0; i < len(result); i++ {
+		dtoResults[i] = MigrateDataResponseItemDTO{
+			Type:   MigrateDataType(result[i].Type),
+			RefID:  result[i].RefID,
+			Status: ItemStatus(result[i].Status),
+			Error:  result[i].Error,
+		}
+	}
+
+	respDto := GetSnapshotResponseDTO{
+		SnapshotDTO: SnapshotDTO{
+			SnapshotUID: snapshot.UID,
+			Status:      fromSnapshotStatus(snapshot.Status),
+			SessionUID:  sessUid,
+			Created:     snapshot.Created,
+			Finished:    snapshot.Finished,
+		},
+		Results: dtoResults,
+	}
+
+	return response.JSON(http.StatusOK, respDto)
+}
+
+// swagger:route GET /cloudmigration/migration/{uid}/snapshots migrations getShapshotList
+//
+// Get a list of snapshots for a session.
+//
+// Responses:
+// 200: snapshotListResponse
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 500: internalServerError
+func (cma *CloudMigrationAPI) GetSnapshotList(c *contextmodel.ReqContext) response.Response {
+	ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.GetShapshotList")
+	defer span.End()
+
+	uid := web.Params(c.Req)[":uid"]
+	if err := util.ValidateUID(uid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
+	}
+	q := cloudmigration.ListSnapshotsQuery{
+		SessionUID: uid,
+		Limit:      c.QueryInt("limit"),
+		Offset:     c.QueryInt("offset"),
+	}
+	if q.Limit == 0 {
+		q.Limit = 100
+	}
+
+	snapshotList, err := cma.cloudMigrationService.GetSnapshotList(ctx, q)
+	if err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "error retrieving snapshot list", err)
+	}
+
+	dtos := make([]SnapshotDTO, len(snapshotList))
+	for i := 0; i < len(snapshotList); i++ {
+		dtos[i] = SnapshotDTO{
+			SnapshotUID: snapshotList[i].UID,
+			Status:      fromSnapshotStatus(snapshotList[i].Status),
+			SessionUID:  uid,
+			Created:     snapshotList[i].Created,
+			Finished:    snapshotList[i].Finished,
+		}
+	}
+
+	return response.JSON(http.StatusOK, SnapshotListResponseDTO{
+		Snapshots: dtos,
+	})
+}
+
+// swagger:route POST /cloudmigration/migration/{uid}/snapshot/{snapshotUid}/upload migrations uploadSnapshot
+//
+// Upload a snapshot to the Grafana Migration Service for processing.
+//
+// Responses:
+// 200:
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 500: internalServerError
+func (cma *CloudMigrationAPI) UploadSnapshot(c *contextmodel.ReqContext) response.Response {
+	ctx, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.UploadSnapshot")
+	defer span.End()
+
+	sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
+	if err := util.ValidateUID(sessUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
+	}
+	if err := util.ValidateUID(snapshotUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
+	}
+
+	if err := cma.cloudMigrationService.UploadSnapshot(ctx, sessUid, snapshotUid); err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "error uploading snapshot", err)
+	}
+
+	return response.JSON(http.StatusOK, nil)
+}
+
+// swagger:route POST /cloudmigration/migration/{uid}/snapshot/{snapshotUid}/cancel migrations cancelSnapshot
+//
+// Cancel a snapshot, wherever it is in its processing chain.
+// TODO: Implement
+//
+// Responses:
+// 200:
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 500: internalServerError
+func (cma *CloudMigrationAPI) CancelSnapshot(c *contextmodel.ReqContext) response.Response {
+	_, span := cma.tracer.Start(c.Req.Context(), "MigrationAPI.CancelSnapshot")
+	defer span.End()
+
+	sessUid, snapshotUid := web.Params(c.Req)[":uid"], web.Params(c.Req)[":snapshotUid"]
+	if err := util.ValidateUID(sessUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid session uid", err)
+	}
+	if err := util.ValidateUID(snapshotUid); err != nil {
+		return response.ErrOrFallback(http.StatusBadRequest, "invalid snapshot uid", err)
+	}
+
+	// Implement
+
+	return response.JSON(http.StatusOK, nil)
+}

pkg/services/cloudmigration/api/curl_commands.txt

@@ -0,0 +1,21 @@
+[sample token] // NOT A REAL TOKEN
+eyJUb2tlbiI6ImNvbXBsZXRlbHlfZmFrZV90b2tlbl9jZG9peTFhYzdwdXlwZCIsIkluc3RhbmNlIjp7IlN0YWNrSUQiOjEyMzQ1LCJTbHVnIjoic3R1Ymluc3RhbmNlIiwiUmVnaW9uU2x1ZyI6ImZha2UtcmVnaW9uIiwiQ2x1c3RlclNsdWciOiJmYWtlLWNsdXNlciJ9fQ==
+
+[create session}
+curl -X POST -H "Content-Type: application/json" \
+  http://admin:admin@localhost:3000/api/cloudmigration/migration \
+  -d '{"AuthToken":"eyJUb2tlbiI6ImNvbXBsZXRlbHlfZmFrZV90b2tlbl9jZG9peTFhYzdwdXlwZCIsIkluc3RhbmNlIjp7IlN0YWNrSUQiOjEyMzQ1LCJTbHVnIjoic3R1Ymluc3RhbmNlIiwiUmVnaW9uU2x1ZyI6ImZha2UtcmVnaW9uIiwiQ2x1c3RlclNsdWciOiJmYWtlLWNsdXNlciJ9fQ=="}'
+
+[create snapshot]
+curl -X POST -H "Content-Type: application/json" \
+  http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot
+
+[get snapshot list]
+curl -X GET http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshots?limit=100&offset=0
+
+[get snapshot]
+curl -X GET http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot/{snapshotUid}
+
+[upload snapshot]
+curl -X POST -H "Content-Type: application/json" \
+   http://admin:admin@localhost:3000/api/cloudmigration/migration/{sessionUid}/snapshot/{snapshotUid}/upload

pkg/services/cloudmigration/api/dtos.go

@@ -38,7 +38,6 @@ type CreateAccessTokenResponseDTO struct {
 // swagger:parameters deleteCloudMigrationToken
 type DeleteCloudMigrationToken struct {
 	// UID of a cloud migration token
-	//
 	// in: path
 	UID string `json:"uid"`
 }
@@ -67,7 +66,6 @@ type CloudMigrationSessionListResponseDTO struct {
 // swagger:parameters getSession
 type GetCloudMigrationSessionRequest struct {
 	// UID of a migration session
-	//
 	// in: path
 	UID string `json:"uid"`
 }
@@ -92,7 +90,6 @@ type CloudMigrationSessionRequestDTO struct {
 // swagger:parameters runCloudMigration
 type RunCloudMigrationRequest struct {
 	// UID of a migration
-	//
 	// in: path
 	UID string `json:"uid"`
 }
@@ -138,7 +135,6 @@ const (
 // swagger:parameters getCloudMigrationRun
 type GetMigrationRunParams struct {
 	// RunUID of a migration run
-	//
 	// in: path
 	RunUID string `json:"runUID"`
 }
@@ -146,7 +142,6 @@ type GetMigrationRunParams struct {
 // swagger:parameters getCloudMigrationRunList
 type GetCloudMigrationRunList struct {
 	// UID of a migration
-	//
 	// in: path
 	UID string `json:"uid"`
 }
@@ -154,10 +149,10 @@ type GetCloudMigrationRunList struct {
 // swagger:response cloudMigrationRunListResponse
 type CloudMigrationRunListResponse struct {
 	// in: body
-	Body SnapshotListDTO
+	Body CloudMigrationRunListDTO
 }
 
-type SnapshotListDTO struct {
+type CloudMigrationRunListDTO struct {
 	Runs []MigrateDataResponseListDTO `json:"runs"`
 }
 
@@ -168,7 +163,6 @@ type MigrateDataResponseListDTO struct {
 // swagger:parameters deleteSession
 type DeleteMigrationSessionRequest struct {
 	// UID of a migration session
-	//
 	// in: path
 	UID string `json:"uid"`
 }
@@ -207,3 +201,138 @@ func convertMigrateDataResponseToDTO(r cloudmigration.MigrateDataResponse) Migra
 		Items:  items,
 	}
 }
+
+// Base snapshot without results
+type SnapshotDTO struct {
+	SnapshotUID string         `json:"uid"`
+	Status      SnapshotStatus `json:"status"`
+	SessionUID  string         `json:"sessionUid"`
+	Created     time.Time      `json:"created"`
+	Finished    time.Time      `json:"finished"`
+}
+
+// swagger:enum SnapshotStatus
+type SnapshotStatus string
+
+const (
+	SnapshotStatusInitializing      SnapshotStatus = "INITIALIZING"
+	SnapshotStatusCreating          SnapshotStatus = "CREATING"
+	SnapshotStatusPendingUpload     SnapshotStatus = "PENDING_UPLOAD"
+	SnapshotStatusUploading         SnapshotStatus = "UPLOADING"
+	SnapshotStatusPendingProcessing SnapshotStatus = "PENDING_PROCESSING"
+	SnapshotStatusProcessing        SnapshotStatus = "PROCESSING"
+	SnapshotStatusFinished          SnapshotStatus = "FINISHED"
+	SnapshotStatusError             SnapshotStatus = "ERROR"
+	SnapshotStatusUnknown           SnapshotStatus = "UNKNOWN"
+)
+
+func fromSnapshotStatus(status cloudmigration.SnapshotStatus) SnapshotStatus {
+	switch status {
+	case cloudmigration.SnapshotStatusInitializing:
+		return SnapshotStatusInitializing
+	case cloudmigration.SnapshotStatusCreating:
+		return SnapshotStatusCreating
+	case cloudmigration.SnapshotStatusPendingUpload:
+		return SnapshotStatusPendingUpload
+	case cloudmigration.SnapshotStatusUploading:
+		return SnapshotStatusUploading
+	case cloudmigration.SnapshotStatusPendingProcessing:
+		return SnapshotStatusPendingProcessing
+	case cloudmigration.SnapshotStatusProcessing:
+		return SnapshotStatusProcessing
+	case cloudmigration.SnapshotStatusFinished:
+		return SnapshotStatusFinished
+	case cloudmigration.SnapshotStatusError:
+		return SnapshotStatusError
+	default:
+		return SnapshotStatusUnknown
+	}
+}
+
+// swagger:parameters createSnapshot
+type CreateSnapshotRequest struct {
+	// UID of a session
+	// in: path
+	UID string `json:"uid"`
+}
+
+// swagger:response createSnapshotResponse
+type CreateSnapshotResponse struct {
+	// in: body
+	Body CreateSnapshotResponseDTO
+}
+
+type CreateSnapshotResponseDTO struct {
+	SnapshotUID string `json:"uid"`
+}
+
+// swagger:parameters getSnapshot
+type GetSnapshotParams struct {
+	// Session UID of a session
+	// in: path
+	UID string `json:"uid"`
+
+	// UID of a snapshot
+	// in: path
+	SnapshotUID string `json:"snapshotUid"`
+}
+
+// swagger:response getSnapshotResponse
+type GetSnapshotResponse struct {
+	// in: body
+	Body GetSnapshotResponseDTO
+}
+
+type GetSnapshotResponseDTO struct {
+	SnapshotDTO
+	Results []MigrateDataResponseItemDTO `json:"results"`
+}
+
+// swagger:parameters getShapshotList
+type GetSnapshotListParams struct {
+	// Offset is used for pagination with limit
+	// in:query
+	// required:false
+	// default: 0
+	Offset int `json:"offset"`
+	// Max limit for results returned.
+	// in:query
+	// required:false
+	// default: 100
+	Limit int `json:"limit"`
+	// Session UID of a session
+	// in: path
+	UID string `json:"uid"`
+}
+
+// swagger:response snapshotListResponse
+type SnapshotListResponse struct {
+	// in: body
+	Body SnapshotListResponseDTO
+}
+
+type SnapshotListResponseDTO struct {
+	Snapshots []SnapshotDTO `json:"snapshots"`
+}
+
+// swagger:parameters uploadSnapshot
+type UploadSnapshotParams struct {
+	// Session UID of a session
+	// in: path
+	UID string `json:"uid"`
+
+	// UID of a snapshot
+	// in: path
+	SnapshotUID string `json:"snapshotUid"`
+}
+
+// swagger:parameters cancelSnapshot
+type CancelSnapshotParams struct {
+	// Session UID of a session
+	// in: path
+	UID string `json:"uid"`
+
+	// UID of a snapshot
+	// in: path
+	SnapshotUID string `json:"snapshotUid"`
+}