CloudMigrations: Fix issues with snapshot resource limits (#105425)
* fix bulk inserts * commit progress so cursor doesn't sabotage me * add more tests * get everything working * rename variable * update comment * regen mocks, fix k8s list method maybe * fix bug with duplicate entries * lint * Snapshots: Use slices.Chunk for batching inserts * remove extra linebreak --------- Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>
This commit is contained in:
Michael Mandrus
co-authored by
Matheus Macabu
Matheus Macabu
parent
8ee1f2c1fc
commit
6205e126cc
@@ -3,9 +3,11 @@ package cloudmigrationimpl
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/cloudmigration"
|
||||
fakeSecrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
|
||||
@@ -96,25 +98,6 @@ func Test_GetMigrationSessionByUID(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
/** rewrite this test using the new functions
|
||||
func Test_DeleteMigrationSession(t *testing.T) {
|
||||
_, s := setUpTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("deletes a session from the db", func(t *testing.T) {
|
||||
uid := "qwerty"
|
||||
session, snapshots, err := s.DeleteMigrationSessionByUID(ctx, uid)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uid, session.UID)
|
||||
require.NotNil(t, snapshots)
|
||||
|
||||
// now we try to find it, should return an error
|
||||
_, err = s.GetMigrationSessionByUID(ctx, uid)
|
||||
require.ErrorIs(t, cloudmigration.ErrMigrationNotFound, err)
|
||||
})
|
||||
}
|
||||
*/
|
||||
|
||||
func Test_SnapshotManagement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -182,6 +165,92 @@ func Test_SnapshotManagement(t *testing.T) {
|
||||
require.ErrorIs(t, err, cloudmigration.ErrSnapshotNotFound)
|
||||
require.Nil(t, snapshot)
|
||||
})
|
||||
|
||||
t.Run("tests a snapshot with a large number of resources", func(t *testing.T) {
|
||||
session, err := s.CreateMigrationSession(ctx, cloudmigration.CloudMigrationSession{
|
||||
OrgID: 1,
|
||||
AuthToken: encodeToken("token"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// create a snapshot
|
||||
snapshotUid, err := s.CreateSnapshot(ctx, cloudmigration.CloudMigrationSnapshot{
|
||||
SessionUID: session.UID,
|
||||
Status: cloudmigration.SnapshotStatusCreating,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, snapshotUid)
|
||||
|
||||
// Generate 50,001 test resources in order to test both update conditions (reached the batch limit or reached the end)
|
||||
const numResources = 50001
|
||||
resources := make([]cloudmigration.CloudMigrationResource, numResources)
|
||||
|
||||
for i := 0; i < numResources; i++ {
|
||||
resources[i] = cloudmigration.CloudMigrationResource{
|
||||
Name: fmt.Sprintf("Resource %d", i),
|
||||
Type: cloudmigration.DashboardDataType,
|
||||
RefID: fmt.Sprintf("refid-%d", i),
|
||||
Status: cloudmigration.ItemStatusPending,
|
||||
}
|
||||
}
|
||||
|
||||
// Update the snapshot with the resources to create
|
||||
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
|
||||
UID: snapshotUid,
|
||||
Status: cloudmigration.SnapshotStatusPendingUpload,
|
||||
SessionID: session.UID,
|
||||
LocalResourcesToCreate: resources,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the Snapshot and ensure it's in the right state
|
||||
snapshot, err := s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, cloudmigration.SnapshotStatusPendingUpload, snapshot.Status)
|
||||
require.Len(t, snapshot.Resources, numResources)
|
||||
|
||||
for i, r := range snapshot.Resources {
|
||||
assert.Equal(t, cloudmigration.ItemStatusPending, r.Status)
|
||||
|
||||
if i%2 == 0 {
|
||||
snapshot.Resources[i].Status = cloudmigration.ItemStatusOK
|
||||
} else {
|
||||
snapshot.Resources[i].Status = cloudmigration.ItemStatusError
|
||||
}
|
||||
}
|
||||
|
||||
// Update the snapshot with the resources to update
|
||||
err = s.UpdateSnapshot(ctx, cloudmigration.UpdateSnapshotCmd{
|
||||
UID: snapshotUid,
|
||||
Status: cloudmigration.SnapshotStatusFinished,
|
||||
SessionID: session.UID,
|
||||
CloudResourcesToUpdate: snapshot.Resources,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the Snapshot and ensure it's in the right state
|
||||
snapshot, err = s.GetSnapshotByUID(ctx, 1, session.UID, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, cloudmigration.SnapshotStatusFinished, snapshot.Status)
|
||||
|
||||
for i, r := range snapshot.Resources {
|
||||
if i%2 == 0 {
|
||||
assert.Equal(t, cloudmigration.ItemStatusOK, r.Status)
|
||||
} else {
|
||||
assert.Equal(t, cloudmigration.ItemStatusError, r.Status)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func Test_SnapshotResources(t *testing.T) {
|
||||
@@ -357,6 +426,116 @@ func Test_SnapshotResources(t *testing.T) {
|
||||
assert.Equal(t, "2", results[0].UID)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test creating and updating a large number of resources", func(t *testing.T) {
|
||||
// Generate 50,001 test resources in order to test both update conditions (reached the batch limit or reached the end)
|
||||
const numResources = 50001
|
||||
resources := make([]cloudmigration.CloudMigrationResource, numResources)
|
||||
snapshotUid := uuid.New().String()
|
||||
|
||||
t.Run("create the resources", func(t *testing.T) {
|
||||
for i := 0; i < numResources; i++ {
|
||||
resources[i] = cloudmigration.CloudMigrationResource{
|
||||
Name: fmt.Sprintf("Resource %d", i),
|
||||
Type: cloudmigration.DashboardDataType,
|
||||
RefID: fmt.Sprintf("refid-%d", i),
|
||||
Status: cloudmigration.ItemStatusPending,
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to create all resources at once -- it should batch under the hood
|
||||
err := s.CreateSnapshotResources(ctx, snapshotUid, resources)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the resources and ensure they're all there
|
||||
resources, err := s.getSnapshotResources(ctx, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, resources, numResources)
|
||||
})
|
||||
|
||||
t.Run("update the resources", func(t *testing.T) {
|
||||
// Initially, update with a mix of ok and error statuses
|
||||
for i := 0; i < numResources; i++ {
|
||||
if i%2 == 0 {
|
||||
resources[i].Status = cloudmigration.ItemStatusOK
|
||||
} else {
|
||||
resources[i].Status = cloudmigration.ItemStatusError
|
||||
resources[i].ErrorCode = "test-error"
|
||||
resources[i].Error = "test-error-message"
|
||||
}
|
||||
}
|
||||
|
||||
err := s.UpdateSnapshotResources(ctx, snapshotUid, resources)
|
||||
require.NoError(t, err)
|
||||
|
||||
resources, err := s.getSnapshotResources(ctx, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, resources, numResources)
|
||||
for i, r := range resources {
|
||||
if i%2 == 0 {
|
||||
assert.Equal(t, cloudmigration.ItemStatusOK, r.Status)
|
||||
} else {
|
||||
assert.Equal(t, cloudmigration.ItemStatusError, r.Status)
|
||||
assert.Equal(t, "test-error", string(r.ErrorCode))
|
||||
assert.Equal(t, "test-error-message", r.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// Now update with only error statuses
|
||||
for i := 0; i < numResources; i++ {
|
||||
resources[i].Status = cloudmigration.ItemStatusError
|
||||
resources[i].ErrorCode = "test-error-2"
|
||||
resources[i].Error = "test-error-message-2"
|
||||
}
|
||||
|
||||
err = s.UpdateSnapshotResources(ctx, snapshotUid, resources)
|
||||
require.NoError(t, err)
|
||||
|
||||
resources, err = s.getSnapshotResources(ctx, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, resources, numResources)
|
||||
for _, r := range resources {
|
||||
assert.Equal(t, cloudmigration.ItemStatusError, r.Status)
|
||||
assert.Equal(t, "test-error-2", string(r.ErrorCode))
|
||||
assert.Equal(t, "test-error-message-2", r.Error)
|
||||
}
|
||||
|
||||
// Finally, all okay
|
||||
for i := 0; i < numResources; i++ {
|
||||
resources[i].Status = cloudmigration.ItemStatusOK
|
||||
}
|
||||
|
||||
err = s.UpdateSnapshotResources(ctx, snapshotUid, resources)
|
||||
require.NoError(t, err)
|
||||
|
||||
resources, err = s.getSnapshotResources(ctx, snapshotUid, cloudmigration.SnapshotResultQueryParams{
|
||||
ResultPage: 1,
|
||||
ResultLimit: numResources,
|
||||
SortColumn: cloudmigration.SortColumnID,
|
||||
SortOrder: cloudmigration.SortOrderAsc,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, resources, numResources)
|
||||
for _, r := range resources {
|
||||
assert.Equal(t, cloudmigration.ItemStatusOK, r.Status)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func Test_SnapshotResourceCaseInsensitiveSorting(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user