Storage: Return an iterator from backend List (#91185)

pkg/storage/unified/sql/backend.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"math"
 	"sync"
 	"time"
 
@@ -17,6 +18,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"go.opentelemetry.io/otel/trace/noop"
 	"google.golang.org/protobuf/proto"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 const trace_prefix = "sql.resource."
@@ -323,14 +325,12 @@ func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *
 	return &res.ReadResponse
 }
 
-func (b *backend) PrepareList(ctx context.Context, req *resource.ListRequest) *resource.ListResponse {
+func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
 	_, span := b.tracer.Start(ctx, trace_prefix+"List")
 	defer span.End()
 
 	if req.Options == nil || req.Options.Key.Group == "" || req.Options.Key.Resource == "" {
-		return &resource.ListResponse{
-			Error: resource.NewBadRequestError("missing group or resource"),
-		}
+		return 0, fmt.Errorf("missing group or resource")
 	}
 
 	// TODO: think about how to handler VersionMatch. We should be able to use latest for the first page (only).
@@ -338,21 +338,81 @@ func (b *backend) PrepareList(ctx context.Context, req *resource.ListRequest) *r
 	// TODO: add support for RemainingItemCount
 
 	if req.ResourceVersion > 0 || req.NextPageToken != "" {
-		return b.listAtRevision(ctx, req)
+		return b.listAtRevision(ctx, req, cb)
 	}
-	return b.listLatest(ctx, req)
+	return b.listLatest(ctx, req, cb)
 }
 
+type listIter struct {
+	rows   *sql.Rows
+	offset int64
+	listRV int64
+
+	// any error
+	err error
+
+	// The row
+	rv        int64
+	value     []byte
+	namespace string
+	name      string
+}
+
+// ContinueToken implements resource.ListIterator.
+func (l *listIter) ContinueToken() string {
+	return ContinueToken{ResourceVersion: l.listRV, StartOffset: l.offset}.String()
+}
+
+// Error implements resource.ListIterator.
+func (l *listIter) Error() error {
+	return l.err
+}
+
+// Name implements resource.ListIterator.
+func (l *listIter) Name() string {
+	return l.name
+}
+
+// Namespace implements resource.ListIterator.
+func (l *listIter) Namespace() string {
+	return l.namespace
+}
+
+// ResourceVersion implements resource.ListIterator.
+func (l *listIter) ResourceVersion() int64 {
+	return l.rv
+}
+
+// Value implements resource.ListIterator.
+func (l *listIter) Value() []byte {
+	return l.value
+}
+
+// Next implements resource.ListIterator.
+func (l *listIter) Next() bool {
+	if l.rows.Next() {
+		l.offset++
+		l.err = l.rows.Scan(&l.rv, &l.namespace, &l.name, &l.value)
+		return true
+	}
+	return false
+}
+
+var _ resource.ListIterator = (*listIter)(nil)
+
 // listLatest fetches the resources from the resource table.
-func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest) *resource.ListResponse {
-	out := &resource.ListResponse{
-		ResourceVersion: 0,
+func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
+	if req.NextPageToken != "" {
+		return 0, fmt.Errorf("only works for the first page")
+	}
+	if req.ResourceVersion > 0 {
+		return 0, fmt.Errorf("only works for the 'latest' resource version")
 	}
 
+	iter := &listIter{}
 	err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
 		var err error
-
-		out.ResourceVersion, err = fetchLatestRV(ctx, tx, b.dialect, req.Options.Key.Group, req.Options.Key.Resource)
+		iter.listRV, err = fetchLatestRV(ctx, tx, b.dialect, req.Options.Key.Group, req.Options.Key.Resource)
 		if err != nil {
 			return err
 		}
@@ -360,96 +420,78 @@ func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest) *re
 		listReq := sqlResourceListRequest{
 			SQLTemplate: sqltemplate.New(b.dialect),
 			Request:     new(resource.ListRequest),
-			Response:    new(resource.ResourceWrapper),
 		}
 		listReq.Request = proto.Clone(req).(*resource.ListRequest)
-		if req.Limit > 0 {
-			listReq.Request.Limit++ // fetch one extra row for Limit
-		}
 
-		items, err := dbutil.Query(ctx, tx, sqlResourceList, listReq)
+		rows, err := dbutil.QueryRows(ctx, tx, sqlResourceList, listReq)
+		if rows != nil {
+			defer func() {
+				if err := rows.Close(); err != nil {
+					b.log.Warn("listLatest error closing rows", "error", err)
+				}
+			}()
+		}
 		if err != nil {
-			return fmt.Errorf("list latest resources: %w", err)
+			return err
 		}
 
-		if 0 < req.Limit && int(req.Limit) < len(items) {
-			// remove the additional item we added synthetically above
-			clear(items[req.Limit:])
-			items = items[:req.Limit]
-
-			out.NextPageToken = ContinueToken{
-				ResourceVersion: out.ResourceVersion,
-				StartOffset:     req.Limit,
-			}.String()
-		}
-		out.Items = items
-
-		return nil
+		iter.rows = rows
+		return cb(iter)
 	})
-	if err != nil {
-		out.Error = resource.AsErrorResult(err)
-	}
-	return out
+	return iter.listRV, err
 }
 
 // listAtRevision fetches the resources from the resource_history table at a specific revision.
-func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest) *resource.ListResponse {
+func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
 	// Get the RV
-	rv := req.ResourceVersion
-	offset := int64(0)
+	iter := &listIter{listRV: req.ResourceVersion}
 	if req.NextPageToken != "" {
 		continueToken, err := GetContinueToken(req.NextPageToken)
 		if err != nil {
-			return &resource.ListResponse{
-				Error: resource.AsErrorResult(fmt.Errorf("get continue token: %w", err)),
-			}
+			return 0, fmt.Errorf("get continue token: %w", err)
 		}
-		rv = continueToken.ResourceVersion
-		offset = continueToken.StartOffset
-	}
+		iter.listRV = continueToken.ResourceVersion
+		iter.offset = continueToken.StartOffset
 
-	out := &resource.ListResponse{
-		ResourceVersion: rv,
+		if req.ResourceVersion != 0 && req.ResourceVersion != iter.listRV {
+			return 0, apierrors.NewBadRequest("request resource version does not math token")
+		}
+	}
+	if iter.listRV < 1 {
+		return 0, apierrors.NewBadRequest("expecting an explicit resource version query")
 	}
 
 	err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
+		limit := int64(0) // ignore limit
+		if iter.offset > 0 {
+			limit = math.MaxInt64 // a limit is required for offset
+		}
 		listReq := sqlResourceHistoryListRequest{
 			SQLTemplate: sqltemplate.New(b.dialect),
 			Request: &historyListRequest{
-				ResourceVersion: rv,
-				Limit:           req.Limit,
-				Offset:          offset,
+				ResourceVersion: iter.listRV,
+				Limit:           limit,
+				Offset:          iter.offset,
 				Options:         req.Options,
 			},
-			Response: new(resource.ResourceWrapper),
-		}
-		if listReq.Request.Limit > 0 {
-			listReq.Request.Limit++ // fetch one extra row for Limit
 		}
 
-		items, err := dbutil.Query(ctx, tx, sqlResourceHistoryList, listReq)
+		rows, err := dbutil.QueryRows(ctx, tx, sqlResourceHistoryList, listReq)
+		if rows != nil {
+			defer func() {
+				if err := rows.Close(); err != nil {
+					b.log.Warn("listAtRevision error closing rows", "error", err)
+				}
+			}()
+		}
 		if err != nil {
-			return fmt.Errorf("list resources at revision: %w", err)
+			return err
 		}
 
-		if 0 < req.Limit && int(req.Limit) < len(items) {
-			// remove the additional item we added synthetically above
-			clear(items[req.Limit:])
-			items = items[:req.Limit]
-
-			out.NextPageToken = ContinueToken{
-				ResourceVersion: out.ResourceVersion,
-				StartOffset:     req.Limit + offset,
-			}.String()
-		}
-		out.Items = items
-
-		return nil
+		iter.rows = rows
+		return cb(iter)
 	})
-	if err != nil {
-		out.Error = resource.AsErrorResult(err)
-	}
-	return out
+	return iter.listRV, err
 }
 
 func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.WrittenEvent, error) {
@@ -616,7 +658,7 @@ func (b *backend) poll(ctx context.Context, grp string, res string, since int64,
 // in a single roundtrip. This would reduce the latency of the operation, and also increase the
 // throughput of the system. This is a good candidate for a future optimization.
 func resourceVersionAtomicInc(ctx context.Context, x db.ContextExecer, d sqltemplate.Dialect, key *resource.ResourceKey) (newVersion int64, err error) {
-	// TODO: refactor this code to run in a multi-statement transaction in order to minimise the number of roundtrips.
+	// TODO: refactor this code to run in a multi-statement transaction in order to minimize the number of round trips.
 	// 1 Lock the row for update
 	rv, err := dbutil.QueryRow(ctx, x, sqlResourceVersionGet, sqlResourceVersionRequest{
 		SQLTemplate:     sqltemplate.New(d),
@@ -656,6 +698,6 @@ func resourceVersionAtomicInc(ctx context.Context, x db.ContextExecer, d sqltemp
 		return 0, fmt.Errorf("increase resource version: %w", err)
 	}
 
-	// 3. Retun the incremended value
+	// 3. Return the incremented value
 	return nextRV, nil
 }

pkg/storage/unified/sql/data/resource_history_list.sql

@@ -1,6 +1,8 @@
 SELECT
-    kv.{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
-    kv.{{ .Ident "value" | .Into .Response.Value }}
+    kv.{{ .Ident "resource_version" }},
+    kv.{{ .Ident "namespace" }},
+    kv.{{ .Ident "name" }},
+    kv.{{ .Ident "value" }}
     FROM {{ .Ident "resource_history" }} as kv 
     INNER JOIN  (
         SELECT {{ .Ident "namespace" }}, {{ .Ident "group" }}, {{ .Ident "resource" }}, {{ .Ident "name" }},  max({{ .Ident "resource_version" }}) AS {{ .Ident "resource_version" }}

pkg/storage/unified/sql/data/resource_list.sql

@@ -1,6 +1,8 @@
 SELECT
-    {{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
-    {{ .Ident "value" | .Into .Response.Value }}
+    {{ .Ident "resource_version" }},
+    {{ .Ident "namespace" }},
+    {{ .Ident "name" }},
+    {{ .Ident "value" }}
     FROM {{ .Ident "resource" }}
     WHERE 1 = 1
         {{ if and .Request.Options .Request.Options.Key }}
@@ -18,7 +20,4 @@ SELECT
             {{ end }}
         {{ end }}
     ORDER BY {{ .Ident "resource_version" }} DESC
-    {{ if (gt .Request.Limit 0) }}
-    LIMIT {{ .Arg .Request.Limit }}
-    {{ end }}
 ;

pkg/storage/unified/sql/dbutil/dbutil.go

@@ -107,10 +107,8 @@ func Exec(ctx context.Context, x db.ContextExecer, tmpl *template.Template, req
 }
 
 // Query uses `req` as input for a single-statement, set-returning query
-// generated with `tmpl`, and executed in `x`. The `Results` method of `req`
-// should return a deep copy since it will be used multiple times to decode each
-// value. It returns an error if more than one result set is returned.
-func Query[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.WithResults[T]) ([]T, error) {
+// generated with `tmpl`, and executed in `x`.
+func QueryRows(ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.SQLTemplateIface) (*sql.Rows, error) {
 	if err := req.Validate(); err != nil {
 		return nil, fmt.Errorf("Query: invalid request for template %q: %w",
 			tmpl.Name(), err)
@@ -134,6 +132,18 @@ func Query[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Templa
 			RawQuery:     rawQuery,
 		}
 	}
+	return rows, err
+}
+
+// Query uses `req` as input for a single-statement, set-returning query
+// generated with `tmpl`, and executed in `x`. The `Results` method of `req`
+// should return a deep copy since it will be used multiple times to decode each
+// value. It returns an error if more than one result set is returned.
+func Query[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.WithResults[T]) ([]T, error) {
+	rows, err := QueryRows(ctx, x, tmpl, req)
+	if err != nil {
+		return nil, err
+	}
 
 	var ret []T
 	for rows.Next() {

pkg/storage/unified/sql/queries.go

@@ -114,26 +114,13 @@ func (r sqlResourceReadRequest) Validate() error {
 // List
 type sqlResourceListRequest struct {
 	*sqltemplate.SQLTemplate
-	Request  *resource.ListRequest
-	Response *resource.ResourceWrapper
+	Request *resource.ListRequest
 }
 
 func (r sqlResourceListRequest) Validate() error {
 	return nil // TODO
 }
 
-func (r sqlResourceListRequest) Results() (*resource.ResourceWrapper, error) {
-	// sqlResourceListRequest is a set-returning query. As such, it
-	// should not return its *Response, since that will be overwritten in the
-	// next call to `Scan`, so it needs to return a copy of it. Note, though,
-	// that it is safe to return the same `Response.Value` since `Scan`
-	// allocates a new slice of bytes each time.
-	return &resource.ResourceWrapper{
-		ResourceVersion: r.Response.ResourceVersion,
-		Value:           r.Response.Value,
-	}, nil
-}
-
 type historyListRequest struct {
 	ResourceVersion, Limit, Offset int64
 	Options                        *resource.ListOptions

pkg/storage/unified/sql/queries_test.go

@@ -159,7 +159,6 @@ func TestQueries(t *testing.T) {
 							},
 						},
 					},
-					Response: new(resource.ResourceWrapper),
 				},
 				Expected: expected{
 					"resource_list_mysql_sqlite.sql": dialects{

pkg/storage/unified/sql/test/integration_test.go

@@ -26,7 +26,7 @@ func TestMain(m *testing.M) {
 	testsuite.Run(m)
 }
 
-func newServer(t *testing.T) sql.Backend {
+func newServer(t *testing.T) (sql.Backend, resource.ResourceServer) {
 	t.Helper()
 
 	dbstore := infraDB.InitTestDB(t)
@@ -48,7 +48,15 @@ func newServer(t *testing.T) sql.Backend {
 	err = ret.Init(testutil.NewDefaultTestContext(t))
 	require.NoError(t, err)
 
-	return ret
+	server, err := resource.NewResourceServer(resource.ResourceServerOptions{
+		Backend:     ret,
+		Diagnostics: ret,
+		Lifecycle:   ret,
+	})
+	require.NoError(t, err)
+	require.NotNil(t, server)
+
+	return ret, server
 }
 
 func TestIntegrationBackendHappyPath(t *testing.T) {
@@ -57,57 +65,65 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
 		t.Skip("skipping integration test")
 	}
 
-	ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
-	store := newServer(t)
+	testUserA := &identity.StaticRequester{
+		Type:           identity.TypeUser,
+		Login:          "testuser",
+		UserID:         123,
+		UserUID:        "u123",
+		OrgRole:        identity.RoleAdmin,
+		IsGrafanaAdmin: true, // can do anything
+	}
+	ctx := identity.WithRequester(context.Background(), testUserA)
+	backend, server := newServer(t)
 
-	stream, err := store.WatchWriteEvents(ctx)
+	stream, err := backend.WatchWriteEvents(context.Background()) // Using a different context to avoid canceling the stream after the DefaultContextTimeout
 	require.NoError(t, err)
 
 	t.Run("Add 3 resources", func(t *testing.T) {
-		rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
+		rv, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED)
 		require.NoError(t, err)
 		require.Equal(t, int64(1), rv)
 
-		rv, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
+		rv, err = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED)
 		require.NoError(t, err)
 		require.Equal(t, int64(2), rv)
 
-		rv, err = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED)
+		rv, err = writeEvent(ctx, backend, "item3", resource.WatchEvent_ADDED)
 		require.NoError(t, err)
 		require.Equal(t, int64(3), rv)
 	})
 
 	t.Run("Update item2", func(t *testing.T) {
-		rv, err := writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED)
+		rv, err := writeEvent(ctx, backend, "item2", resource.WatchEvent_MODIFIED)
 		require.NoError(t, err)
 		require.Equal(t, int64(4), rv)
 	})
 
 	t.Run("Delete item1", func(t *testing.T) {
-		rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_DELETED)
+		rv, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_DELETED)
 		require.NoError(t, err)
 		require.Equal(t, int64(5), rv)
 	})
 
 	t.Run("Read latest item 2", func(t *testing.T) {
-		resp := store.ReadResource(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
-		require.NoError(t, err)
+		resp := backend.ReadResource(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
+		require.Nil(t, resp.Error)
 		require.Equal(t, int64(4), resp.ResourceVersion)
 		require.Equal(t, "item2 MODIFIED", string(resp.Value))
 	})
 
-	t.Run("Read early verion of item2", func(t *testing.T) {
-		resp := store.ReadResource(ctx, &resource.ReadRequest{
+	t.Run("Read early version of item2", func(t *testing.T) {
+		resp := backend.ReadResource(ctx, &resource.ReadRequest{
 			Key:             resourceKey("item2"),
 			ResourceVersion: 3, // item2 was created at rv=2 and updated at rv=4
 		})
-		require.NoError(t, err)
+		require.Nil(t, resp.Error)
 		require.Equal(t, int64(2), resp.ResourceVersion)
 		require.Equal(t, "item2 ADDED", string(resp.Value))
 	})
 
 	t.Run("PrepareList latest", func(t *testing.T) {
-		resp := store.PrepareList(ctx, &resource.ListRequest{
+		resp, err := server.List(ctx, &resource.ListRequest{
 			Options: &resource.ListOptions{
 				Key: &resource.ResourceKey{
 					Namespace: "namespace",
@@ -117,6 +133,7 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
 			},
 		})
 		require.NoError(t, err)
+		require.Nil(t, resp.Error)
 		require.Len(t, resp.Items, 2)
 		require.Equal(t, "item2 MODIFIED", string(resp.Items[0].Value))
 		require.Equal(t, "item3 ADDED", string(resp.Items[1].Value))
@@ -157,42 +174,42 @@ func TestIntegrationBackendWatchWriteEventsFromLastest(t *testing.T) {
 	}
 
 	ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
-	store := newServer(t)
+	backend, _ := newServer(t)
 
 	// Create a few resources before initing the watch
-	_, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
+	_, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED)
 	require.NoError(t, err)
 
 	// Start the watch
-	stream, err := store.WatchWriteEvents(ctx)
+	stream, err := backend.WatchWriteEvents(ctx)
 	require.NoError(t, err)
 
 	// Create one more event
-	_, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
+	_, err = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED)
 	require.NoError(t, err)
 	require.Equal(t, "item2", (<-stream).Key.Name)
 }
 
-func TestIntegrationBackendPrepareList(t *testing.T) {
+func TestIntegrationBackendList(t *testing.T) {
 	t.Skip("TODO: test blocking, skipping to unblock Enterprise until we fix this")
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
 
 	ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
-	store := newServer(t)
+	backend, server := newServer(t)
 
-	// Create a few resources before initing the watch
-	_, _ = writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)    // rv=1
-	_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)    // rv=2 - will be modified at rv=6
-	_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED)    // rv=3 - will be deleted at rv=7
-	_, _ = writeEvent(ctx, store, "item4", resource.WatchEvent_ADDED)    // rv=4
-	_, _ = writeEvent(ctx, store, "item5", resource.WatchEvent_ADDED)    // rv=5
-	_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED) // rv=6
-	_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_DELETED)  // rv=7
-	_, _ = writeEvent(ctx, store, "item6", resource.WatchEvent_ADDED)    // rv=8
+	// Create a few resources before starting the watch
+	_, _ = writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED)    // rv=1
+	_, _ = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED)    // rv=2 - will be modified at rv=6
+	_, _ = writeEvent(ctx, backend, "item3", resource.WatchEvent_ADDED)    // rv=3 - will be deleted  at rv=7
+	_, _ = writeEvent(ctx, backend, "item4", resource.WatchEvent_ADDED)    // rv=4
+	_, _ = writeEvent(ctx, backend, "item5", resource.WatchEvent_ADDED)    // rv=5
+	_, _ = writeEvent(ctx, backend, "item2", resource.WatchEvent_MODIFIED) // rv=6
+	_, _ = writeEvent(ctx, backend, "item3", resource.WatchEvent_DELETED)  // rv=7
+	_, _ = writeEvent(ctx, backend, "item6", resource.WatchEvent_ADDED)    // rv=8
 	t.Run("fetch all latest", func(t *testing.T) {
-		res := store.PrepareList(ctx, &resource.ListRequest{
+		res, err := server.List(ctx, &resource.ListRequest{
 			Options: &resource.ListOptions{
 				Key: &resource.ResourceKey{
 					Group:    "group",
@@ -200,6 +217,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 				},
 			},
 		})
+		require.NoError(t, err)
 		require.Nil(t, res.Error)
 		require.Len(t, res.Items, 5)
 		// should be sorted by resource version DESC
@@ -213,7 +231,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 	})
 
 	t.Run("list latest first page ", func(t *testing.T) {
-		res := store.PrepareList(ctx, &resource.ListRequest{
+		res, err := server.List(ctx, &resource.ListRequest{
 			Limit: 3,
 			Options: &resource.ListOptions{
 				Key: &resource.ResourceKey{
@@ -222,6 +240,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 				},
 			},
 		})
+		require.NoError(t, err)
 		require.Nil(t, res.Error)
 		require.Len(t, res.Items, 3)
 		continueToken, err := sql.GetContinueToken(res.NextPageToken)
@@ -230,11 +249,10 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 		require.Equal(t, "item2 MODIFIED", string(res.Items[1].Value))
 		require.Equal(t, "item5 ADDED", string(res.Items[2].Value))
 		require.Equal(t, int64(8), continueToken.ResourceVersion)
-		require.Equal(t, int64(3), continueToken.StartOffset)
 	})
 
 	t.Run("list at revision", func(t *testing.T) {
-		res := store.PrepareList(ctx, &resource.ListRequest{
+		res, err := server.List(ctx, &resource.ListRequest{
 			ResourceVersion: 4,
 			Options: &resource.ListOptions{
 				Key: &resource.ResourceKey{
@@ -243,6 +261,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 				},
 			},
 		})
+		require.NoError(t, err)
 		require.Nil(t, res.Error)
 		require.Len(t, res.Items, 4)
 		require.Equal(t, "item4 ADDED", string(res.Items[0].Value))
@@ -253,7 +272,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 	})
 
 	t.Run("fetch first page at revision with limit", func(t *testing.T) {
-		res := store.PrepareList(ctx, &resource.ListRequest{
+		res, err := server.List(ctx, &resource.ListRequest{
 			Limit:           3,
 			ResourceVersion: 7,
 			Options: &resource.ListOptions{
@@ -263,6 +282,8 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 				},
 			},
 		})
+		require.NoError(t, err)
+		require.NoError(t, err)
 		require.Nil(t, res.Error)
 		require.Len(t, res.Items, 3)
 		t.Log(res.Items)
@@ -273,7 +294,6 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 		continueToken, err := sql.GetContinueToken(res.NextPageToken)
 		require.NoError(t, err)
 		require.Equal(t, int64(7), continueToken.ResourceVersion)
-		require.Equal(t, int64(3), continueToken.StartOffset)
 	})
 
 	t.Run("fetch second page at revision", func(t *testing.T) {
@@ -281,7 +301,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 			ResourceVersion: 8,
 			StartOffset:     2,
 		}
-		res := store.PrepareList(ctx, &resource.ListRequest{
+		res, err := server.List(ctx, &resource.ListRequest{
 			NextPageToken: continueToken.String(),
 			Limit:         2,
 			Options: &resource.ListOptions{
@@ -291,12 +311,14 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
 				},
 			},
 		})
+		require.NoError(t, err)
 		require.Nil(t, res.Error)
 		require.Len(t, res.Items, 2)
+		t.Log(res.Items)
 		require.Equal(t, "item5 ADDED", string(res.Items[0].Value))
 		require.Equal(t, "item4 ADDED", string(res.Items[1].Value))
 
-		continueToken, err := sql.GetContinueToken(res.NextPageToken)
+		continueToken, err = sql.GetContinueToken(res.NextPageToken)
 		require.NoError(t, err)
 		require.Equal(t, int64(8), continueToken.ResourceVersion)
 		require.Equal(t, int64(4), continueToken.StartOffset)

pkg/storage/unified/sql/testdata/resource_history_list_mysql_sqlite.sql

@@ -1,4 +1,4 @@
-SELECT kv."resource_version", kv."value"
+SELECT kv."resource_version", kv."namespace", kv."name", kv."value"
 FROM "resource_history" as kv
 INNER JOIN (
 	SELECT "namespace", "group", "resource", "name", max("resource_version") AS "resource_version"

pkg/storage/unified/sql/testdata/resource_list_mysql_sqlite.sql

@@ -1,6 +1,5 @@
-SELECT "resource_version", "value"
+SELECT "resource_version", "namespace", "name", "value"
     FROM "resource"
     WHERE 1 = 1 AND "namespace" = ?
     ORDER BY "resource_version" DESC
-    LIMIT ?
 ;