public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CloudMigrations: Store encryption key in unified secrets table (#90908)

Browse Source 
* store encryption key in unified secrets table

* fix local dev mode

* make metadata more realistic

* fix tests

* fix sql queries against postgres

* fix stats endpoint
This commit is contained in:
Michael Mandrus
2024-07-25 06:34:25 +03:00
committed by GitHub
parent 49c756d774
commit dc355331a6
11 changed files with 60 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/api/api.go
+1
View File
@@ -445,6 +445,7 @@ func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.R
dtoStats := SnapshotResourceStats{
Types: make(map[MigrateDataType]int, len(snapshot.StatsRollup.CountsByStatus)),
Statuses: make(map[ItemStatus]int, len(snapshot.StatsRollup.CountsByType)),
Total: snapshot.StatsRollup.Total,
}
for s, c := range snapshot.StatsRollup.CountsByStatus {
dtoStats.Statuses[ItemStatus(s)] = c
pkg/services/cloudmigration/api/api_test.go
+1 -1
View File
@@ -471,7 +471,7 @@ func TestCloudMigrationAPI_GetSnapshot(t *testing.T) {
requestUrl: "/api/cloudmigration/migration/1234/snapshot/1",
basicRole: org.RoleAdmin,
expectedHttpResult: http.StatusOK,
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{}}}`,
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{},"total":0}}`,
},
{
desc: "should return 403 if no used is not admin",
pkg/services/cloudmigration/api/dtos.go
+1
View File
@@ -307,6 +307,7 @@ type GetSnapshotResponseDTO struct {
type SnapshotResourceStats struct {
Types map[MigrateDataType]int `json:"types"`
Statuses map[ItemStatus]int `json:"statuses"`
Total int `json:"total"`
}
// swagger:parameters getShapshotList
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go
+3 -1
View File
@@ -28,6 +28,7 @@ import (
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/gcom"
"github.com/grafana/grafana/pkg/services/secrets"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
@@ -82,6 +83,7 @@ func ProvideService(
features featuremgmt.FeatureToggles,
db db.DB,
dsService datasources.DataSourceService,
secretsStore secretskv.SecretsKVStore,
secretsService secrets.Service,
routeRegister routing.RouteRegister,
prom prometheus.Registerer,
@@ -95,7 +97,7 @@ func ProvideService(
}
s := &Service{
store: &sqlStore{db: db, secretsService: secretsService},
store: &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
log: log.New(LogPrefix),
cfg: cfg,
features: features,
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
+2
View File
@@ -24,6 +24,7 @@ import (
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/folder/foldertest"
secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
@@ -438,6 +439,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi
featuremgmt.FlagDashboardRestore),
sqlStore,
dsService,
secretskv.NewFakeSQLSecretsKVStore(t),
secretsService,
rr,
prometheus.DefaultRegisterer,
pkg/services/cloudmigration/cloudmigrationimpl/xorm_store.go
+20 -30
View File
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/secrets"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util"
)
@@ -17,11 +18,13 @@ var _ store = (*sqlStore)(nil)
type sqlStore struct {
db db.DB
secretsStore secretskv.SecretsKVStore
secretsService secrets.Service
}
const (
tableName = "cloud_migration_resource"
tableName = "cloud_migration_resource"
secretType = "cloudmigration-snapshot-encryption-key"
)
func (ss *sqlStore) GetMigrationSessionByUID(ctx context.Context, uid string) (*cloudmigration.CloudMigrationSession, error) {
@@ -157,14 +160,14 @@ func (ss *sqlStore) GetMigrationStatusList(ctx context.Context, migrationUID str
}
func (ss *sqlStore) CreateSnapshot(ctx context.Context, snapshot cloudmigration.CloudMigrationSnapshot) (string, error) {
if err := ss.encryptKey(ctx, &snapshot); err != nil {
return "", err
}
if snapshot.UID == "" {
snapshot.UID = util.GenerateShortUID()
}
if err := ss.secretsStore.Set(ctx, secretskv.AllOrganizations, snapshot.UID, secretType, string(snapshot.EncryptionKey)); err != nil {
return "", err
}
err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
snapshot.Created = time.Now()
snapshot.Updated = time.Now()
@@ -228,8 +231,12 @@ func (ss *sqlStore) GetSnapshotByUID(ctx context.Context, uid string, resultPage
return nil, err
}
if err := ss.decryptKey(ctx, &snapshot); err != nil {
if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
return &snapshot, err
} else if !found {
return &snapshot, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
} else {
snapshot.EncryptionKey = []byte(secret)
}
resources, err := ss.GetSnapshotResources(ctx, uid, resultPage, resultLimit)
@@ -259,8 +266,12 @@ func (ss *sqlStore) GetSnapshotList(ctx context.Context, query cloudmigration.Li
return nil, err
}
for i, snapshot := range snapshots {
if err := ss.decryptKey(ctx, &snapshot); err != nil {
if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
return nil, err
} else if !found {
return nil, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
} else {
snapshot.EncryptionKey = []byte(secret)
}
if stats, err := ss.GetSnapshotResourceStats(ctx, snapshot.UID); err != nil {
@@ -346,14 +357,14 @@ func (ss *sqlStore) GetSnapshotResourceStats(ctx context.Context, snapshotUid st
} else {
total = int(t)
}
sess.Select("count(uid) as 'count', resource_type as 'type'").
sess.Select("count(uid) as \"count\", resource_type as \"type\"").
Table(tableName).
GroupBy("type").
Where("snapshot_uid = ?", snapshotUid)
if err := sess.Find(&typeCounts); err != nil {
return err
}
sess.Select("count(uid) as 'count', status").
sess.Select("count(uid) as \"count\", status").
Table(tableName).
GroupBy("status").
Where("snapshot_uid = ?", snapshotUid)
@@ -411,24 +422,3 @@ func (ss *sqlStore) decryptToken(ctx context.Context, cm *cloudmigration.CloudMi
return nil
}
func (ss *sqlStore) encryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
s, err := ss.secretsService.Encrypt(ctx, snapshot.EncryptionKey, secrets.WithoutScope())
if err != nil {
return fmt.Errorf("encrypting key: %w", err)
}
snapshot.EncryptionKey = s
return nil
}
func (ss *sqlStore) decryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
t, err := ss.secretsService.Decrypt(ctx, snapshot.EncryptionKey)
if err != nil {
return fmt.Errorf("decrypting key: %w", err)
}
snapshot.EncryptionKey = t
return nil
}
pkg/services/cloudmigration/cloudmigrationimpl/xorm_store_test.go
+2
View File
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/cloudmigration"
fakeSecrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/tests/testsuite"
"github.com/grafana/grafana/pkg/util"
@@ -274,6 +275,7 @@ func setUpTest(t *testing.T) (*sqlstore.SQLStore, *sqlStore) {
s := &sqlStore{
db: testDB,
secretsService: fakeSecrets.FakeSecretsService{},
secretsStore: secretskv.NewFakeSQLSecretsKVStore(t),
}
ctx := context.Background()
pkg/services/cloudmigration/gmsclient/inmemory_client.go
+21 -2
View File
@@ -2,6 +2,7 @@ package gmsclient
import (
"context"
"encoding/json"
"fmt"
"math/rand"
@@ -51,16 +52,34 @@ func (c *memoryClientImpl) MigrateData(
return &result, nil
}
func (c *memoryClientImpl) StartSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
func (c *memoryClientImpl) StartSnapshot(_ context.Context, sess cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
publicKey, _, err := box.GenerateKey(cryptoRand.Reader)
if err != nil {
return nil, fmt.Errorf("nacl: generating public and private key: %w", err)
}
snapshotUid := uuid.NewString()
metadataBuffer, err := json.Marshal(struct {
SnapshotID string `json:"snapshotID"`
StackID string `json:"stackID"`
Slug string `json:"slug"`
}{
SnapshotID: snapshotUid,
StackID: fmt.Sprintf("%d", sess.StackID),
Slug: sess.Slug,
})
if err != nil {
return nil, fmt.Errorf("marshalling metadata: %w", err)
}
c.snapshot = &cloudmigration.StartSnapshotResponse{
EncryptionKey: publicKey[:],
SnapshotID: uuid.NewString(),
SnapshotID: snapshotUid,
MaxItemsPerPartition: 10,
Algo: "nacl",
Metadata: metadataBuffer,
}
return c.snapshot, nil
pkg/services/cloudmigration/model.go
+1 -1
View File
@@ -37,7 +37,7 @@ type CloudMigrationSnapshot struct {
UID string `xorm:"uid"`
SessionUID string `xorm:"session_uid"`
Status SnapshotStatus
EncryptionKey []byte `xorm:"encryption_key"` // stored in the unified secrets table
EncryptionKey []byte `xorm:"-"` // stored in the unified secrets table
LocalDir string `xorm:"local_directory"`
GMSSnapshotUID string `xorm:"gms_snapshot_uid"`
ErrorString string `xorm:"error_string"`