From c9e4c26c118803ef0f85e6e3d8dc64ec52fb4cfc Mon Sep 17 00:00:00 2001 From: Will Assis <35489495+gassiss@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:36:02 -0500 Subject: [PATCH] unified-storage: add more list pagination tests (#113543) * unified-storage: add more list pagination tests --- .../unified/resource/storage_backend_test.go | 40 ++++++++++++++-- .../unified/testing/storage_backend.go | 48 ++++++++++--------- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/pkg/storage/unified/resource/storage_backend_test.go b/pkg/storage/unified/resource/storage_backend_test.go index 772362aceef..85cf3331d7d 100644 --- a/pkg/storage/unified/resource/storage_backend_test.go +++ b/pkg/storage/unified/resource/storage_backend_test.go @@ -537,21 +537,51 @@ func TestKvStorageBackend_ListIterator_WithPagination(t *testing.T) { var continueToken2 string _, err = backend.ListIterator(ctx, listReq, func(iter ListIterator) error { + count := 0 for iter.Next() { if err := iter.Error(); err != nil { return err } secondPageItems = append(secondPageItems, iter.Name()) + count++ + // Simulate pagination by getting continue token after limit items + if count >= int(listReq.Limit) { + continueToken2 = iter.ContinueToken() + break + } } - // Capture continue token for potential third page - continueToken2 = iter.ContinueToken() return iter.Error() }) - // TODO: fix the ListIterator to respect the limit. This require a change to the resource server. require.NoError(t, err) - require.Equal(t, 3, len(secondPageItems)) - require.Equal(t, []string{"resource-3", "resource-4", "resource-5"}, secondPageItems) + require.Equal(t, 2, len(secondPageItems)) + require.Equal(t, []string{"resource-3", "resource-4"}, secondPageItems) require.NotEmpty(t, continueToken2) + + // third page using continue token + listReq.NextPageToken = continueToken2 + var thirdPageItems []string + var continueToken3 string + + _, err = backend.ListIterator(ctx, listReq, func(iter ListIterator) error { + count := 0 + for iter.Next() { + if err := iter.Error(); err != nil { + return err + } + thirdPageItems = append(thirdPageItems, iter.Name()) + count++ + // Simulate pagination by getting continue token after limit items + if count >= int(listReq.Limit) { + continueToken = iter.ContinueToken() + break + } + } + return iter.Error() + }) + require.NoError(t, err) + require.Equal(t, 1, len(thirdPageItems)) + require.Equal(t, []string{"resource-5"}, thirdPageItems) + require.Empty(t, continueToken3) } func TestKvStorageBackend_ListIterator_EmptyResult(t *testing.T) { backend := setupTestStorageBackend(t) diff --git a/pkg/storage/unified/testing/storage_backend.go b/pkg/storage/unified/testing/storage_backend.go index 440f2af8a26..18be19d407e 100644 --- a/pkg/storage/unified/testing/storage_backend.go +++ b/pkg/storage/unified/testing/storage_backend.go @@ -411,7 +411,7 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend require.Empty(t, res.NextPageToken) }) - t.Run("list latest first page ", func(t *testing.T) { + t.Run("fetch all latest with pagination", func(t *testing.T) { res, err := server.List(ctx, &resourcepb.ListRequest{ Limit: 3, Options: &resourcepb.ListOptions{ @@ -431,6 +431,22 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED") require.Contains(t, string(res.Items[2].Value), "item4 ADDED") require.GreaterOrEqual(t, continueToken.ResourceVersion, rv8) + + res, err = server.List(ctx, &resourcepb.ListRequest{ + Limit: 3, + NextPageToken: continueToken.String(), + Options: &resourcepb.ListOptions{ + Key: &resourcepb.ResourceKey{ + Namespace: ns, + Group: "group", + Resource: "resource", + }, + }, + }) + require.NoError(t, err) + require.Nil(t, res.Error) + require.Len(t, res.Items, 2) + require.Empty(t, res.NextPageToken) }) t.Run("list at revision", func(t *testing.T) { @@ -454,7 +470,7 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend require.Empty(t, res.NextPageToken) }) - t.Run("fetch first page at revision with limit", func(t *testing.T) { + t.Run("list at revision with pagination", func(t *testing.T) { res, err := server.List(ctx, &resourcepb.ListRequest{ Limit: 3, ResourceVersion: rv7, @@ -470,7 +486,6 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend require.NoError(t, err) require.Nil(t, res.Error) require.Len(t, res.Items, 3) - t.Log(res.Items) require.Contains(t, string(res.Items[0].Value), "item1 ADDED") require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED") require.Contains(t, string(res.Items[2].Value), "item4 ADDED") @@ -478,17 +493,11 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend continueToken, err := resource.GetContinueToken(res.NextPageToken) require.NoError(t, err) require.Equal(t, rv7, continueToken.ResourceVersion) - }) - t.Run("fetch second page at revision", func(t *testing.T) { - continueToken := &resource.ContinueToken{ - ResourceVersion: rv8, - StartOffset: 2, - SortAscending: false, - } - res, err := server.List(ctx, &resourcepb.ListRequest{ - NextPageToken: continueToken.String(), - Limit: 2, + res, err = server.List(ctx, &resourcepb.ListRequest{ + Limit: 3, + ResourceVersion: rv7, + NextPageToken: continueToken.String(), Options: &resourcepb.ListOptions{ Key: &resourcepb.ResourceKey{ Namespace: ns, @@ -498,16 +507,11 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend }, }) require.NoError(t, err) - require.Nil(t, res.Error) - require.Len(t, res.Items, 2) - t.Log(res.Items) - require.Contains(t, string(res.Items[0].Value), "item4 ADDED") - require.Contains(t, string(res.Items[1].Value), "item5 ADDED") - - continueToken, err = resource.GetContinueToken(res.NextPageToken) require.NoError(t, err) - require.Equal(t, rv8, continueToken.ResourceVersion) - require.Equal(t, int64(4), continueToken.StartOffset) + require.Nil(t, res.Error) + require.Len(t, res.Items, 1) + require.Contains(t, string(res.Items[0].Value), "item5 ADDED") + require.Empty(t, res.NextPageToken) }) }