feat: legacy ListIterator with batches (#115038)

* feat: legacy ListIterator with batches

* chore: address code review

* chore: remove nil check in nextBatch

* chore: move close before count check

* chore: add err field to batchingIterator for its own errors

* chore: remove unused import

pkg/registry/apis/dashboard/legacy/queries_test.go

@@ -100,6 +100,42 @@ func TestDashboardQueries(t *testing.T) {
 						Order:         "ASC",
 					}),
 				},
+				{
+					// Tests that MaxRows generates LIMIT clause for regular dashboard queries
+					Name: "dashboard_with_max_rows",
+					Data: getQuery(&DashboardQuery{
+						OrgID:   2,
+						MaxRows: 100,
+					}),
+				},
+				{
+					// Tests that MaxRows generates LIMIT clause for history queries
+					Name: "history_with_max_rows",
+					Data: getQuery(&DashboardQuery{
+						OrgID:      1,
+						GetHistory: true,
+						MaxRows:    50,
+					}),
+				},
+				{
+					// Tests that MaxRows + LastID generates correct pagination query
+					Name: "dashboard_with_max_rows_last_id",
+					Data: getQuery(&DashboardQuery{
+						OrgID:   2,
+						MaxRows: 100,
+						LastID:  500,
+					}),
+				},
+				{
+					// Tests that MaxRows + LastID generates correct pagination query
+					Name: "history_with_max_rows_last_id",
+					Data: getQuery(&DashboardQuery{
+						OrgID:      2,
+						MaxRows:    100,
+						GetHistory: true,
+						LastID:     500,
+					}),
+				},
 			},
 			sqlQueryPanels: {
 				{

pkg/registry/apis/dashboard/legacy/query_dashboards.sql

@@ -71,6 +71,9 @@ WHERE dashboard.is_folder = {{ .Arg .Query.GetFolders }}
     COALESCE(dashboard_version.version, dashboard.version) {{ .Query.Order }},
     {{ end }}
     dashboard.uid ASC
+  {{ if .Query.MaxRows }}
+  LIMIT {{ .Arg .Query.MaxRows }}
+  {{ end }}
   {{ else }}
   {{ if .Query.UID }}
   AND dashboard.uid = {{ .Arg .Query.UID }}
@@ -83,4 +86,7 @@ WHERE dashboard.is_folder = {{ .Arg .Query.GetFolders }}
   AND dashboard.deleted IS NULL
   {{ end }}
   ORDER BY dashboard.id DESC
+  {{ if .Query.MaxRows }}
+  LIMIT {{ .Arg .Query.MaxRows }}
+  {{ end }}
   {{ end }}

pkg/registry/apis/dashboard/legacy/sql_dashboards.go

@@ -712,6 +712,128 @@ func (r *rowsWrapper) Value() []byte {
 	return b
 }
 
+// batchingIterator wraps rowsWrapper to fetch data in batches
+type batchingIterator struct {
+	wrapper   *rowsWrapper
+	a         *dashboardSqlAccess
+	ctx       context.Context
+	helper    *legacysql.LegacyDatabaseHelper
+	query     *DashboardQuery
+	batchSize int
+	done      bool
+	err       error
+}
+
+var _ resource.ListIterator = (*batchingIterator)(nil)
+
+func (b *batchingIterator) Error() error {
+	if b.err != nil {
+		return b.err
+	}
+	return b.wrapper.Error()
+}
+
+func (b *batchingIterator) ContinueToken() string {
+	return b.wrapper.ContinueToken()
+}
+
+func (b *batchingIterator) ResourceVersion() int64 {
+	return b.wrapper.ResourceVersion()
+}
+
+func (b *batchingIterator) Namespace() string {
+	return b.wrapper.Namespace()
+}
+
+func (b *batchingIterator) Name() string {
+	return b.wrapper.Name()
+}
+
+func (b *batchingIterator) Folder() string {
+	return b.wrapper.Folder()
+}
+
+func (b *batchingIterator) Value() []byte {
+	return b.wrapper.Value()
+}
+
+func (b *batchingIterator) Close() error {
+	return b.wrapper.Close()
+}
+
+func newBatchingIterator(ctx context.Context, a *dashboardSqlAccess, helper *legacysql.LegacyDatabaseHelper, query *DashboardQuery) (*batchingIterator, error) {
+	iter := &batchingIterator{
+		a:         a,
+		ctx:       ctx,
+		helper:    helper,
+		query:     query,
+		batchSize: query.MaxRows,
+	}
+
+	// Loads the first batch
+	if err := iter.nextBatch(query.LastID); err != nil {
+		return nil, err
+	}
+	return iter, nil
+}
+
+func (b *batchingIterator) nextBatch(lastID int64) error {
+	b.query.LastID = lastID
+	wrapper, err := b.a.getRows(b.ctx, b.helper, b.query)
+	if err != nil {
+		return err
+	}
+	b.wrapper = wrapper
+	return nil
+}
+
+func (b *batchingIterator) Next() bool {
+	if b.done {
+		return false
+	}
+
+	// Try to get next row from current batch
+	if b.wrapper.Next() {
+		return true
+	}
+
+	// Check for errors in current wrapper
+	if b.Error() != nil {
+		return false
+	}
+
+	// No more rows in current batch - close it
+	if err := b.wrapper.Close(); err != nil {
+		// Should not happen, but handle it
+		b.err = err
+		b.done = true
+		return false
+	}
+
+	// Current batch exhausted - check if we got a full batch (might be more data)
+	if b.wrapper.count < b.batchSize {
+		// Got fewer rows than batch size, so we're done
+		b.done = true
+		return false
+	}
+
+	// Fetch next batch with LastID from last row
+	if err := b.nextBatch(b.wrapper.row.token.id); err != nil {
+		b.err = err
+		b.done = true
+		return false
+	}
+
+	// Try to get first row from new batch
+	if b.wrapper.Next() {
+		return true
+	}
+
+	// New batch is empty, we're done
+	b.done = true
+	return false
+}
+
 func generateFallbackDashboard(data []byte, title, uid string) ([]byte, error) {
 	generatedDashboard := map[string]interface{}{
 		"editable": true,

pkg/registry/apis/dashboard/legacy/storage.go

@@ -305,11 +305,19 @@ func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resourcepb.L
 		return 0, fmt.Errorf("token and orgID mismatch")
 	}
 
+	// Default batch size for iterator - fetch rows in batches to avoid slow queries
+	const defaultMaxRows = 500
+	maxRows := defaultMaxRows
+	if req.Limit > 0 && req.Limit < int64(defaultMaxRows) {
+		maxRows = int(req.Limit)
+	}
+
 	query := &DashboardQuery{
-		OrgID:  info.OrgID,
-		Limit:  int(req.Limit),
-		LastID: token.id,
-		Labels: req.Options.Labels,
+		OrgID:   info.OrgID,
+		Limit:   int(req.Limit),
+		MaxRows: maxRows,
+		LastID:  token.id,
+		Labels:  req.Options.Labels,
 	}
 
 	sql, err := a.sql(ctx)
@@ -332,14 +340,15 @@ func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resourcepb.L
 		return 0, err
 	}
 	listRV *= 1000 // Convert to microseconds
-	rows, err := a.getRows(ctx, sql, query)
-	if rows != nil {
+
+	iter, err := newBatchingIterator(ctx, a, sql, query)
+	if iter != nil {
 		defer func() {
-			_ = rows.Close()
+			_ = iter.Close()
 		}()
 	}
 	if err == nil {
-		err = cb(rows)
+		err = cb(iter)
 	}
 	return listRV, err
 }

pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_with_max_rows.sql

@@ -0,0 +1,31 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM `grafana`.`dashboard` as dashboard
+LEFT OUTER JOIN `grafana`.`dashboard_provisioning` as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN `grafana`.`user` as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_with_max_rows_last_id.sql

@@ -0,0 +1,32 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM `grafana`.`dashboard` as dashboard
+LEFT OUTER JOIN `grafana`.`dashboard_provisioning` as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN `grafana`.`user` as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.id < 500
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_with_max_rows.sql

@@ -0,0 +1,34 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM `grafana`.`dashboard` as dashboard
+LEFT OUTER JOIN `grafana`.`dashboard_version` as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN `grafana`.`dashboard_provisioning` as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN `grafana`.`user` as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 1
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 50

pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_with_max_rows_last_id.sql

@@ -0,0 +1,35 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM `grafana`.`dashboard` as dashboard
+LEFT OUTER JOIN `grafana`.`dashboard_version` as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN `grafana`.`dashboard_provisioning` as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN `grafana`.`user` as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard_version.version < 500
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_with_max_rows.sql

@@ -0,0 +1,31 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_with_max_rows_last_id.sql

@@ -0,0 +1,32 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.id < 500
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_with_max_rows.sql

@@ -0,0 +1,34 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 1
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 50

pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_with_max_rows_last_id.sql

@@ -0,0 +1,35 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard_version.version < 500
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_with_max_rows.sql

@@ -0,0 +1,31 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_with_max_rows_last_id.sql

@@ -0,0 +1,32 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard.updated,
+  updated_user.uid       as updated_by,
+  dashboard.updated_by   as updated_by_id,
+  dashboard.version,
+  '' as message,
+  dashboard.data,
+  dashboard.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard.id < 500
+  AND dashboard.deleted IS NULL
+  ORDER BY dashboard.id DESC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_with_max_rows.sql

@@ -0,0 +1,34 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 1
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 50

pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_with_max_rows_last_id.sql

@@ -0,0 +1,35 @@
+SELECT
+  dashboard.org_id,
+  dashboard.id,
+  dashboard.uid,
+  dashboard.title,
+  dashboard.folder_uid,
+  dashboard.deleted,
+  plugin_id,
+  provisioning.name        as repo_name,
+  provisioning.external_id as repo_path,
+  provisioning.check_sum   as repo_hash,
+  provisioning.updated     as repo_ts,
+  dashboard.created,
+  created_user.uid         as created_by,
+  dashboard.created_by     as created_by_id,
+  dashboard_version.created as updated,
+  updated_user.uid       as updated_by,
+  dashboard_version.created_by as updated_by_id,
+  dashboard_version.version,
+  dashboard_version.message,
+  dashboard_version.data,
+  dashboard_version.api_version
+FROM "grafana"."dashboard" as dashboard
+LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id
+LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id
+LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id
+LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard_version.created_by = updated_user.id
+WHERE dashboard.is_folder = FALSE
+  AND dashboard.org_id = 2
+  AND dashboard_version.version < 500
+  ORDER BY
+    dashboard_version.created DESC,
+    dashboard_version.version DESC,
+    dashboard.uid ASC
+  LIMIT 100

pkg/registry/apis/dashboard/legacy/types.go

@@ -16,6 +16,11 @@ type DashboardQuery struct {
 	UID   string // to select a single dashboard
 	Limit int
 
+	// MaxRows is used internally by the iterator to fetch data in batches
+	// When set, the SQL query will include LIMIT MaxRows
+	// If Limit is smaller, that will be used instead
+	MaxRows int
+
 	// Included in the continue token
 	// This is the ID from the last dashboard sent in the previous page
 	LastID int64

pkg/tests/apis/dashboard/dashboards_test.go

@@ -293,6 +293,199 @@ func TestIntegrationLegacySupport(t *testing.T) {
 	require.Equal(t, dashboardV0.VERSION, rsp.Result.Meta.APIVersion)
 }
 
+func TestIntegrationListPagination(t *testing.T) {
+	testutil.SkipIntegrationTestInShortMode(t)
+
+	gvr := schema.GroupVersionResource{
+		Group:    dashboardV0.GROUP,
+		Version:  dashboardV0.VERSION,
+		Resource: "dashboards",
+	}
+
+	// Test on modes with legacy
+	modes := []rest.DualWriterMode{rest.Mode1, rest.Mode2, rest.Mode3}
+	for _, mode := range modes {
+		t.Run(fmt.Sprintf("pagination with dual writer mode %d", mode), func(t *testing.T) {
+			helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
+				DisableAnonymous:      true,
+				DisableDataMigrations: true,
+				UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
+					"dashboards.dashboard.grafana.app": {
+						DualWriterMode: mode,
+					},
+				},
+			})
+			t.Cleanup(helper.Shutdown)
+
+			ctx := context.Background()
+			client := helper.GetResourceClient(apis.ResourceClientArgs{
+				User: helper.Org1.Admin,
+				GVR:  gvr,
+			})
+
+			// Test 1: List with no dashboards
+			rsp, err := client.Resource.List(ctx, metav1.ListOptions{})
+			require.NoError(t, err)
+			require.Len(t, rsp.Items, 0)
+
+			// Create 5 dashboards to test pagination with small limits
+			const totalDashboards = 5
+			createdNames := make([]string, 0, totalDashboards)
+			for i := 0; i < totalDashboards; i++ {
+				obj := &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]any{
+							"title":         fmt.Sprintf("Pagination test dashboard %d", i),
+							"schemaVersion": 42,
+						},
+					},
+				}
+				obj.SetGenerateName("pag-")
+				obj.SetAPIVersion(gvr.GroupVersion().String())
+				obj.SetKind("Dashboard")
+				created, err := client.Resource.Create(ctx, obj, metav1.CreateOptions{})
+				require.NoError(t, err)
+				createdNames = append(createdNames, created.GetName())
+			}
+
+			// Test 2: List all without limit - should return all dashboards
+			rsp, err = client.Resource.List(ctx, metav1.ListOptions{})
+			require.NoError(t, err)
+			require.Len(t, rsp.Items, totalDashboards, "should return all %d dashboards", totalDashboards)
+
+			// Test 3: List with small limit (2) - should paginate
+			const pageSize = 2
+			allNames := make(map[string]bool)
+			continueToken := ""
+			pageCount := 0
+
+			for {
+				pageCount++
+				listOpts := metav1.ListOptions{
+					Limit:    pageSize,
+					Continue: continueToken,
+				}
+				rsp, err = client.Resource.List(ctx, listOpts)
+				require.NoError(t, err)
+
+				// Collect names from this page
+				for _, item := range rsp.Items {
+					name := item.GetName()
+					require.False(t, allNames[name], "duplicate item %s found across pages", name)
+					allNames[name] = true
+				}
+
+				// Check if there's more pages
+				continueToken = rsp.GetContinue()
+				if continueToken == "" {
+					break
+				}
+
+				// Safety check to prevent infinite loops
+				require.Less(t, pageCount, 5)
+			}
+
+			// Verify we got all dashboards across all pages
+			require.Len(t, allNames, totalDashboards, "should have collected all %d dashboards across pages", totalDashboards)
+
+			// Verify all created dashboards were found
+			for _, name := range createdNames {
+				require.True(t, allNames[name], "dashboard %s not found in paginated results", name)
+			}
+		})
+
+		t.Run(fmt.Sprintf("history pagination with dual writer mode %d", mode), func(t *testing.T) {
+			helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
+				DisableAnonymous:      true,
+				DisableDataMigrations: true,
+				UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{
+					"dashboards.dashboard.grafana.app": {
+						DualWriterMode: mode,
+					},
+				},
+			})
+			t.Cleanup(helper.Shutdown)
+
+			ctx := context.Background()
+			client := helper.GetResourceClient(apis.ResourceClientArgs{
+				User: helper.Org1.Admin,
+				GVR:  gvr,
+			})
+
+			// Create a dashboard
+			obj := &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"spec": map[string]any{
+						"title":         "History pagination test dashboard",
+						"schemaVersion": 42,
+					},
+				},
+			}
+			obj.SetGenerateName("hist-")
+			obj.SetAPIVersion(gvr.GroupVersion().String())
+			obj.SetKind("Dashboard")
+			created, err := client.Resource.Create(ctx, obj, metav1.CreateOptions{})
+			require.NoError(t, err)
+			dashName := created.GetName()
+
+			// Update the dashboard multiple times to create history entries
+			const totalVersions = 5
+			for i := 1; i < totalVersions; i++ {
+				// Get latest version
+				current, err := client.Resource.Get(ctx, dashName, metav1.GetOptions{})
+				require.NoError(t, err)
+
+				// Update title
+				spec := current.Object["spec"].(map[string]interface{})
+				spec["title"] = fmt.Sprintf("History pagination test dashboard v%d", i+1)
+				current.Object["spec"] = spec
+
+				_, err = client.Resource.Update(ctx, current, metav1.UpdateOptions{})
+				require.NoError(t, err)
+			}
+
+			// Test: List history with pagination
+			labelSelector := utils.LabelKeyGetHistory + "=true"
+			fieldSelector := "metadata.name=" + dashName
+
+			const pageSize int64 = 2
+			allVersions := make([]string, 0)
+			continueToken := ""
+			pageCount := 0
+
+			for {
+				pageCount++
+				listOpts := metav1.ListOptions{
+					LabelSelector: labelSelector,
+					FieldSelector: fieldSelector,
+					Limit:         pageSize,
+					Continue:      continueToken,
+				}
+				rsp, err := client.Resource.List(ctx, listOpts)
+				require.NoError(t, err)
+
+				// Collect resource versions from this page
+				for _, item := range rsp.Items {
+					rv := item.GetResourceVersion()
+					allVersions = append(allVersions, rv)
+				}
+
+				// Check if there's more pages
+				continueToken = rsp.GetContinue()
+				if continueToken == "" {
+					break
+				}
+
+				// Safety check to prevent infinite loops
+				require.Less(t, pageCount, 5)
+			}
+
+			// Verify we got all history versions
+			require.Len(t, allVersions, totalVersions, "should have collected all %d history versions across pages", totalVersions)
+		})
+	}
+}
+
 func TestIntegrationSearchTypeFiltering(t *testing.T) {
 	testutil.SkipIntegrationTestInShortMode(t)