public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cloud migrations: store snapshots in the database (#108551)

Browse Source 
* Cloud migrations: store snapshots in the database

* update github.com/grafana/grafana-cloud-migration-snapshot to v1.9.0

* make update-workspace

* use new field name in test

* return error after call to fmt.Errorf

* create methods for readability / fix session deletiong not deleting snapshots

* remove debugging changes

* update sample.ini

* update tests to include OrgID in ListSnapshotsQuery

* lint

* lint

* Update pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* remove TODO

* Update pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* remove one of the debug logs

---------

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>
This commit is contained in:
Bruno
2025-07-25 11:41:21 -03:00
committed by GitHub
co-authored by Matheus Macabu
parent 47cf7ea8b6
commit b1592b5e36
17 changed files with 470 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/cloudmigration/cloudmigrationimpl/xorm_store_test.go
+63 -16
View File
@@ -1,13 +1,18 @@
package cloudmigrationimpl
import (
"bytes"
"context"
cryptoRand "crypto/rand"
"encoding/base64"
"fmt"
"strconv"
"testing"
"github.com/google/uuid"
snapshot "github.com/grafana/grafana-cloud-migration-snapshot/src"
"github.com/grafana/grafana-cloud-migration-snapshot/src/contracts"
"github.com/grafana/grafana-cloud-migration-snapshot/src/infra/crypto"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/cloudmigration"
fakeSecrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
@@ -15,6 +20,7 @@ import (
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/nacl/box"
)
func Test_GetAllCloudMigrationSessions(t *testing.T) {
@@ -112,17 +118,18 @@ func Test_SnapshotManagement(t *testing.T) {
require.NoError(t, err)
// create a snapshot
uid := uuid.NewString()
cmr := cloudmigration.CloudMigrationSnapshot{
UID: uid,
SessionUID: session.UID,
Status: cloudmigration.SnapshotStatusCreating,
}
snapshotUid, err := s.CreateSnapshot(ctx, cmr)
err = s.CreateSnapshot(ctx, cmr)
require.NoError(t, err)
require.NotEmpty(t, snapshotUid)
//retrieve it from the db
snapshot, err := s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
snapshot, err := s.GetSnapshotByUID(ctx, 1, session.UID, uid, cloudmigration.SnapshotResultQueryParams{
ResultPage: 1,
ResultLimit: 100,
SortColumn: cloudmigration.SortColumnID,
@@ -132,11 +139,11 @@ func Test_SnapshotManagement(t *testing.T) {
require.Equal(t, cloudmigration.SnapshotStatusCreating, snapshot.Status)
// update its status
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{UID: snapshotUid, Status: cloudmigration.SnapshotStatusCreating, SessionID: session.UID})
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{UID: uid, Status: cloudmigration.SnapshotStatusCreating, SessionID: session.UID})
require.NoError(t, err)
//retrieve it again
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, uid, cloudmigration.SnapshotResultQueryParams{
ResultPage: 1,
ResultLimit: 100,
SortColumn: cloudmigration.SortColumnID,
@@ -152,11 +159,11 @@ func Test_SnapshotManagement(t *testing.T) {
require.Equal(t, *snapshot, snapshots[0])
// delete snapshot
err = s.deleteSnapshot(ctx, snapshotUid)
err = s.deleteSnapshot(ctx, uid)
require.NoError(t, err)
// now we expect not to find the snapshot
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, uid, cloudmigration.SnapshotResultQueryParams{
ResultPage: 1,
ResultLimit: 100,
SortColumn: cloudmigration.SortColumnID,
@@ -174,12 +181,13 @@ func Test_SnapshotManagement(t *testing.T) {
require.NoError(t, err)
// create a snapshot
snapshotUid, err := s.CreateSnapshot(ctx, cloudmigration.CloudMigrationSnapshot{
uid := uuid.NewString()
err = s.CreateSnapshot(ctx, cloudmigration.CloudMigrationSnapshot{
UID: uid,
SessionUID: session.UID,
Status: cloudmigration.SnapshotStatusCreating,
})
require.NoError(t, err)
require.NotEmpty(t, snapshotUid)
// Generate 50,001 test resources in order to test both update conditions (reached the batch limit or reached the end)
const numResources = 50001
@@ -196,7 +204,7 @@ func Test_SnapshotManagement(t *testing.T) {
// Update the snapshot with the resources to create
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotUid,
UID: uid,
Status: cloudmigration.SnapshotStatusPendingUpload,
SessionID: session.UID,
LocalResourcesToCreate: resources,
@@ -204,7 +212,7 @@ func Test_SnapshotManagement(t *testing.T) {
require.NoError(t, err)
// Get the Snapshot and ensure it's in the right state
snapshot, err := s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
snapshot, err := s.GetSnapshotByUID(ctx, 1, session.UID, uid, cloudmigration.SnapshotResultQueryParams{
ResultPage: 1,
ResultLimit: numResources,
SortColumn: cloudmigration.SortColumnID,
@@ -226,7 +234,7 @@ func Test_SnapshotManagement(t *testing.T) {
// Update the snapshot with the resources to update
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
UID: snapshotUid,
UID: uid,
Status: cloudmigration.SnapshotStatusFinished,
SessionID: session.UID,
CloudResourcesToUpdate: snapshot.Resources,
@@ -234,7 +242,7 @@ func Test_SnapshotManagement(t *testing.T) {
require.NoError(t, err)
// Get the Snapshot and ensure it's in the right state
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, uid, cloudmigration.SnapshotResultQueryParams{
ResultPage: 1,
ResultLimit: numResources,
SortColumn: cloudmigration.SortColumnID,
@@ -637,7 +645,7 @@ func TestGetSnapshotList(t *testing.T) {
})
t.Run("return no snapshots if limit is set to 0", func(t *testing.T) {
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: sessionUID, Page: 1, Limit: 0})
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: sessionUID, OrgID: 1, Page: 1, Limit: 0})
require.NoError(t, err)
assert.Empty(t, snapshots)
})
@@ -669,7 +677,7 @@ func TestGetSnapshotList(t *testing.T) {
})
t.Run("only the snapshots that belong to a specific session are returned", func(t *testing.T) {
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: "session-uid-that-doesnt-exist", Page: 1, Limit: 100})
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: "session-uid-that-doesnt-exist", OrgID: 1, Page: 1, Limit: 100})
require.NoError(t, err)
assert.Empty(t, snapshots)
})
@@ -680,7 +688,7 @@ func TestGetSnapshotList(t *testing.T) {
require.NoError(t, err)
// Fetch the snapshots that belong to the deleted session.
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: sessionUID, Page: 1, Limit: 100})
snapshots, err := s.GetSnapshotList(ctx, cloudmigration.ListSnapshotsQuery{SessionUID: sessionUID, OrgID: 1, Page: 1, Limit: 100})
require.NoError(t, err)
// No snapshots should be returned because the session that
@@ -801,3 +809,42 @@ func setUpTest(t *testing.T) (*sqlstore.SQLStore, *sqlStore) {
func encodeToken(t string) string {
return base64.StdEncoding.EncodeToString([]byte(t))
}
func TestEncodeDecode(t *testing.T) {
gmsPublicKey, gmsPrivateKey, err := box.GenerateKey(cryptoRand.Reader)
require.NoError(t, err)
grafanaPublicKey, grafanaPrivateKey, err := box.GenerateKey(cryptoRand.Reader)
require.NoError(t, err)
snapshotWriter, err := snapshot.NewSnapshotWriter(contracts.AssymetricKeys{
Public: gmsPublicKey[:],
Private: grafanaPrivateKey[:],
},
crypto.NewNacl(),
"",
)
require.NoError(t, err)
chunk := []snapshot.MigrateDataRequestItemDTO{{
Type: snapshot.AlertRuleGroupType,
RefID: "foo",
Name: "name",
Data: map[string]any{"a": "b"},
}}
encoded, err := snapshotWriter.EncodePartition(chunk)
require.NoError(t, err)
require.NoError(t, snapshotWriter.Write("RESOURCE_TYPE", chunk))
reader := snapshot.NewSnapshotReader(contracts.AssymetricKeys{
Public: grafanaPublicKey[:],
Private: gmsPrivateKey[:],
},
crypto.NewNacl())
partition, err := reader.ReadFile(bytes.NewReader(encoded))
require.NoError(t, err)
require.Equal(t, chunk, partition.Items)
}