public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cloud migration: upload snapshot files using presigned url (#90273)

Browse Source 
* Cloud migration: upload snapshot files using presigned url

* log error if index file cannot be closed

* log error if file cannot be closed in uploadUsingPresignedURL
This commit is contained in:
Bruno
2024-07-11 09:32:02 -03:00
committed by GitHub
parent 69c5fa8361
commit 420ff1d07b
11 changed files with 289 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/gmsclient/gms_client.go
+38 -33
View File
@@ -4,8 +4,11 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/cloudmigration"
@@ -24,11 +27,11 @@ type gmsClientImpl struct {
log *log.ConcreteLogger
}
func (c *gmsClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigrationSession) error {
func (c *gmsClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigrationSession) (err error) {
logger := c.log.FromContext(ctx)
// TODO update service url to gms
path := fmt.Sprintf("https://cms-%s.%s/cloud-migrations/api/v1/validate-key", cm.ClusterSlug, c.domain)
path := fmt.Sprintf("%s/api/v1/validate-key", buildBasePath(c.domain, cm.ClusterSlug))
// validation is an empty POST to GMS with the authorization header included
req, err := http.NewRequest("POST", path, bytes.NewReader(nil))
@@ -45,30 +48,25 @@ func (c *gmsClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.Cloud
logger.Error("error sending http request for token validation", "err", err.Error())
return fmt.Errorf("http request error: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
logger.Error("closing request body", "err", err.Error())
if closeErr := resp.Body.Close(); closeErr != nil {
err = errors.Join(err, fmt.Errorf("closing response body: %w", closeErr))
}
}()
if resp.StatusCode != 200 {
var errResp map[string]any
if err := json.NewDecoder(resp.Body).Decode(&errResp); err != nil {
logger.Error("decoding error response", "err", err.Error())
} else {
return fmt.Errorf("token validation failure: %v", errResp)
}
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("token validation failure: %v", body)
}
return nil
}
func (c *gmsClientImpl) MigrateData(ctx context.Context, cm cloudmigration.CloudMigrationSession, request cloudmigration.MigrateDataRequest) (*cloudmigration.MigrateDataResponse, error) {
func (c *gmsClientImpl) MigrateData(ctx context.Context, cm cloudmigration.CloudMigrationSession, request cloudmigration.MigrateDataRequest) (result *cloudmigration.MigrateDataResponse, err error) {
logger := c.log.FromContext(ctx)
// TODO update service url to gms
path := fmt.Sprintf("https://cms-%s.%s/cloud-migrations/api/v1/migrate-data", cm.ClusterSlug, c.domain)
path := fmt.Sprintf("%s/api/v1/migrate-data", buildBasePath(c.domain, cm.ClusterSlug))
reqDTO := convertRequestToDTO(request)
body, err := json.Marshal(reqDTO)
@@ -90,31 +88,30 @@ func (c *gmsClientImpl) MigrateData(ctx context.Context, cm cloudmigration.Cloud
if err != nil {
c.log.Error("error sending http request for cloud migration run", "err", err.Error())
return nil, fmt.Errorf("http request error: %w", err)
} else if resp.StatusCode >= 400 {
}
defer func() {
if closeErr := resp.Body.Close(); closeErr != nil {
err = errors.Join(err, fmt.Errorf("closing response body: %w", closeErr))
}
}()
if resp.StatusCode >= 400 {
c.log.Error("received error response for cloud migration run", "statusCode", resp.StatusCode)
return nil, fmt.Errorf("http request error: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
logger.Error("closing request body: %w", err)
}
}()
var respDTO MigrateDataResponseDTO
if err := json.NewDecoder(resp.Body).Decode(&respDTO); err != nil {
logger.Error("unmarshalling response body: %w", err)
return nil, fmt.Errorf("unmarshalling migration run response: %w", err)
}
result := convertResponseFromDTO(respDTO)
return &result, nil
res := convertResponseFromDTO(respDTO)
return &res, nil
}
func (c *gmsClientImpl) StartSnapshot(ctx context.Context, session cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
logger := c.log.FromContext(ctx)
path := fmt.Sprintf("https://cms-%s.%s/cloud-migrations/api/v1/start-snapshot", session.ClusterSlug, c.domain)
func (c *gmsClientImpl) StartSnapshot(ctx context.Context, session cloudmigration.CloudMigrationSession) (out *cloudmigration.StartSnapshotResponse, err error) {
path := fmt.Sprintf("%s/api/v1/start-snapshot", buildBasePath(c.domain, session.ClusterSlug))
// Send the request to cms with the associated auth token
req, err := http.NewRequest(http.MethodPost, path, nil)
@@ -130,20 +127,21 @@ func (c *gmsClientImpl) StartSnapshot(ctx context.Context, session cloudmigratio
if err != nil {
c.log.Error("error sending http request to start snapshot", "err", err.Error())
return nil, fmt.Errorf("http request error: %w", err)
} else if resp.StatusCode >= 400 {
c.log.Error("received error response to start snapshot", "statusCode", resp.StatusCode)
return nil, fmt.Errorf("http request error: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
logger.Error("closing request body: %w", err)
if closeErr := resp.Body.Close(); closeErr != nil {
err = errors.Join(err, fmt.Errorf("closing response body: %w", closeErr))
}
}()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(resp.Body)
c.log.Error("received error response to start snapshot", "statusCode", resp.StatusCode)
return nil, fmt.Errorf("http request error: body=%s %w", string(body), err)
}
var result cloudmigration.StartSnapshotResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
logger.Error("unmarshalling response body: %w", err)
return nil, fmt.Errorf("unmarshalling start snapshot response: %w", err)
}
@@ -187,3 +185,10 @@ func convertResponseFromDTO(result MigrateDataResponseDTO) cloudmigration.Migrat
Items: items,
}
}
func buildBasePath(domain, clusterSlug string) string {
if strings.HasPrefix(domain, "http://localhost") {
return domain
}
return fmt.Sprintf("https://cms-%s.%s/cloud-migrations", clusterSlug, domain)
}
pkg/services/cloudmigration/gmsclient/gms_client_test.go
+36
View File
@@ -0,0 +1,36 @@
package gmsclient
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_buildBasePath(t *testing.T) {
t.Parallel()
tests := []struct {
description string
domain string
clusterSlug string
expected string
}{
{
description: "domain starts with http://localhost, should return domain",
domain: "http://localhost:8080",
clusterSlug: "anything",
expected: "http://localhost:8080",
},
{
description: "domain doesn't start with http://localhost, should build a string using the domain and clusterSlug",
domain: "gms-dev",
clusterSlug: "us-east-1",
expected: "https://cms-us-east-1.gms-dev/cloud-migrations",
},
}
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
assert.Equal(t, tt.expected, buildBasePath(tt.domain, tt.clusterSlug))
})
}
}
pkg/services/cloudmigration/gmsclient/inmemory_client.go
+14 -4
View File
@@ -2,11 +2,15 @@ package gmsclient
import (
"context"
"fmt"
"math/rand"
"time"
cryptoRand "crypto/rand"
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/util"
"golang.org/x/crypto/nacl/box"
)
// NewInMemoryClient returns an implementation of Client that returns canned responses
@@ -49,10 +53,16 @@ func (c *memoryClientImpl) MigrateData(
}
func (c *memoryClientImpl) StartSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
publicKey, _, err := box.GenerateKey(cryptoRand.Reader)
if err != nil {
return nil, fmt.Errorf("nacl: generating public and private key: %w", err)
}
c.snapshot = &cloudmigration.StartSnapshotResponse{
EncryptionKey: util.GenerateShortUID(),
SnapshotID: util.GenerateShortUID(),
UploadURL: "localhost:3000",
EncryptionKey: fmt.Sprintf("%x", publicKey[:]),
UploadURL: "localhost:3000",
SnapshotID: uuid.NewString(),
MaxItemsPerPartition: 10,
Algo: "nacl",
}
return c.snapshot, nil