From 6c8d24184a920341c8b2b24c63023ee1ca37dc5f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 24 Feb 2025 11:17:02 +0300 Subject: [PATCH] Provisioning: Export oldest items first (#101189) --- pkg/registry/apis/dashboard/legacy/migrate.go | 1 + pkg/registry/apis/dashboard/legacy/queries.go | 6 +++++ .../apis/dashboard/legacy/queries_test.go | 8 +++++++ .../dashboard/legacy/query_dashboards.sql | 4 ++-- ...--query_dashboards-export_with_history.sql | 22 +++++++++++++++++++ ...--query_dashboards-export_with_history.sql | 22 +++++++++++++++++++ ...--query_dashboards-export_with_history.sql | 22 +++++++++++++++++++ pkg/registry/apis/dashboard/legacy/types.go | 3 +++ 8 files changed, 86 insertions(+), 2 deletions(-) create mode 100755 pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql create mode 100755 pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql create mode 100755 pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql diff --git a/pkg/registry/apis/dashboard/legacy/migrate.go b/pkg/registry/apis/dashboard/legacy/migrate.go index e18150e1cd1..646724b94c8 100644 --- a/pkg/registry/apis/dashboard/legacy/migrate.go +++ b/pkg/registry/apis/dashboard/legacy/migrate.go @@ -195,6 +195,7 @@ func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, OrgID: orgId, Limit: 100000000, GetHistory: opts.WithHistory, // include history + Order: "ASC", // oldest first } blobs := &BlobStoreInfo{} diff --git a/pkg/registry/apis/dashboard/legacy/queries.go b/pkg/registry/apis/dashboard/legacy/queries.go index 9c4688202ba..56807b9cc5a 100644 --- a/pkg/registry/apis/dashboard/legacy/queries.go +++ b/pkg/registry/apis/dashboard/legacy/queries.go @@ -41,10 +41,16 @@ type sqlQuery struct { } func (r sqlQuery) Validate() error { + if r.Query.Order == "ASC" && r.Query.LastID > 0 { + return fmt.Errorf("ascending order does not support paging by last id") + } return nil // TODO } func newQueryReq(sql *legacysql.LegacyDatabaseHelper, query *DashboardQuery) sqlQuery { + if query.Order == "" { + query.Order = "DESC" // use version as RV + } return sqlQuery{ SQLTemplate: sqltemplate.New(sql.DialectForDriver()), Query: query, diff --git a/pkg/registry/apis/dashboard/legacy/queries_test.go b/pkg/registry/apis/dashboard/legacy/queries_test.go index 315c9593450..35fffa9a597 100644 --- a/pkg/registry/apis/dashboard/legacy/queries_test.go +++ b/pkg/registry/apis/dashboard/legacy/queries_test.go @@ -76,6 +76,14 @@ func TestDashboardQueries(t *testing.T) { GetFolders: true, }), }, + { + Name: "export_with_history", + Data: getQuery(&DashboardQuery{ + OrgID: 1, + GetHistory: true, + Order: "ASC", + }), + }, }, sqlQueryPanels: { { diff --git a/pkg/registry/apis/dashboard/legacy/query_dashboards.sql b/pkg/registry/apis/dashboard/legacy/query_dashboards.sql index 80c0b815310..6da03389dab 100644 --- a/pkg/registry/apis/dashboard/legacy/query_dashboards.sql +++ b/pkg/registry/apis/dashboard/legacy/query_dashboards.sql @@ -33,8 +33,8 @@ WHERE dashboard.is_folder = {{ .Arg .Query.GetFolders }} AND dashboard_version.version < {{ .Arg .Query.LastID }} {{ end }} ORDER BY - dashboard_version.created DESC, - dashboard_version.version DESC, + dashboard_version.created {{ .Query.Order }}, + dashboard_version.version {{ .Query.Order }}, dashboard.uid ASC {{ else }} {{ if .Query.UID }} diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql new file mode 100755 index 00000000000..a19e0df94d1 --- /dev/null +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql @@ -0,0 +1,22 @@ +SELECT + dashboard.org_id, dashboard.id, + dashboard.uid, dashboard.folder_uid, + dashboard.deleted, plugin_id, + provisioning.name as repo_name, + provisioning.external_id as repo_path, + provisioning.check_sum as repo_hash, + provisioning.updated as repo_ts, + dashboard.created, created_user.uid as created_by, dashboard.created_by as created_by_id, + dashboard_version.created, updated_user.uid as updated_by,updated_user.id as created_by_id, + dashboard_version.version, dashboard_version.message, dashboard_version.data +FROM `grafana`.`dashboard` as dashboard +LEFT OUTER JOIN `grafana`.`dashboard_version` as dashboard_version ON dashboard.id = dashboard_version.dashboard_id +LEFT OUTER JOIN `grafana`.`dashboard_provisioning` as provisioning ON dashboard.id = provisioning.dashboard_id +LEFT OUTER JOIN `grafana`.`user` as created_user ON dashboard.created_by = created_user.id +LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id +WHERE dashboard.is_folder = FALSE + AND dashboard.org_id = 1 + ORDER BY + dashboard_version.created ASC, + dashboard_version.version ASC, + dashboard.uid ASC diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql new file mode 100755 index 00000000000..8ad23c2206f --- /dev/null +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql @@ -0,0 +1,22 @@ +SELECT + dashboard.org_id, dashboard.id, + dashboard.uid, dashboard.folder_uid, + dashboard.deleted, plugin_id, + provisioning.name as repo_name, + provisioning.external_id as repo_path, + provisioning.check_sum as repo_hash, + provisioning.updated as repo_ts, + dashboard.created, created_user.uid as created_by, dashboard.created_by as created_by_id, + dashboard_version.created, updated_user.uid as updated_by,updated_user.id as created_by_id, + dashboard_version.version, dashboard_version.message, dashboard_version.data +FROM "grafana"."dashboard" as dashboard +LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id +LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id +LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id +LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id +WHERE dashboard.is_folder = FALSE + AND dashboard.org_id = 1 + ORDER BY + dashboard_version.created ASC, + dashboard_version.version ASC, + dashboard.uid ASC diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql new file mode 100755 index 00000000000..8ad23c2206f --- /dev/null +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql @@ -0,0 +1,22 @@ +SELECT + dashboard.org_id, dashboard.id, + dashboard.uid, dashboard.folder_uid, + dashboard.deleted, plugin_id, + provisioning.name as repo_name, + provisioning.external_id as repo_path, + provisioning.check_sum as repo_hash, + provisioning.updated as repo_ts, + dashboard.created, created_user.uid as created_by, dashboard.created_by as created_by_id, + dashboard_version.created, updated_user.uid as updated_by,updated_user.id as created_by_id, + dashboard_version.version, dashboard_version.message, dashboard_version.data +FROM "grafana"."dashboard" as dashboard +LEFT OUTER JOIN "grafana"."dashboard_version" as dashboard_version ON dashboard.id = dashboard_version.dashboard_id +LEFT OUTER JOIN "grafana"."dashboard_provisioning" as provisioning ON dashboard.id = provisioning.dashboard_id +LEFT OUTER JOIN "grafana"."user" as created_user ON dashboard.created_by = created_user.id +LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id +WHERE dashboard.is_folder = FALSE + AND dashboard.org_id = 1 + ORDER BY + dashboard_version.created ASC, + dashboard_version.version ASC, + dashboard.uid ASC diff --git a/pkg/registry/apis/dashboard/legacy/types.go b/pkg/registry/apis/dashboard/legacy/types.go index 8a62ca0c8df..d89655faf52 100644 --- a/pkg/registry/apis/dashboard/legacy/types.go +++ b/pkg/registry/apis/dashboard/legacy/types.go @@ -30,6 +30,9 @@ type DashboardQuery struct { // The label requirements Labels []*resource.Requirement + + // DESC|ASC, how to order the IDs + Order string // asc required to use lastID, desc required for export with history } func (r *DashboardQuery) UseHistoryTable() bool {