public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SQL/Storage: Use xorm code migration for DeletionMarker (#99353)

Browse Source
This commit is contained in:
Ryan McKinley
2025-01-22 14:07:08 +03:00
committed by GitHub
parent 3ae6e09fee
commit 9d30911107
18 changed files with 113 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/sql/backend.go
-6
View File
@@ -106,12 +106,6 @@ func (b *backend) initLocked(ctx context.Context) error {
return fmt.Errorf("no dialect for driver %q", driverName)
}
// Process any data manipulation migrations
err = b.runStartupDataMigrations(ctx)
if err != nil {
return err
}
return b.db.PingContext(ctx)
}
pkg/storage/unified/sql/data/migrator_get_deletion_markers.sql
-9
View File
@@ -1,9 +0,0 @@
SELECT
{{ .Ident "guid" }},
{{ .Ident "value" }},
{{ .Ident "group" }},
{{ .Ident "resource" }},
{{ .Ident "previous_resource_version" }}
FROM {{ .Ident "resource_history" }}
WHERE {{ .Ident "action" }} = 3
AND {{ .Ident "value" }} LIKE {{ .Arg .MarkerQuery }};
pkg/storage/unified/sql/data/migrator_get_value_from_rv.sql
-5
View File
@@ -1,5 +0,0 @@
SELECT {{ .Ident "value" }}
FROM {{ .Ident "resource_history" }}
WHERE {{ .Ident "group" }} = {{ .Arg .Group }}
AND {{ .Ident "resource" }} = {{ .Arg .Resource }}
AND {{ .Ident "resource_version" }} = {{ .Arg .RV }};
pkg/storage/unified/sql/data/migrator_update_value_with_guid.sql
-4
View File
@@ -1,4 +0,0 @@
UPDATE {{ .Ident "resource_history" }}
SET {{ .Ident "value" }} = {{ .Arg .Value }}
WHERE {{ .Ident "guid" }} = {{ .Arg .GUID }}
;
pkg/storage/unified/sql/db/migrations/deletion_markers.go
+111
View File
@@ -0,0 +1,111 @@
package migrations
import (
"fmt"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"xorm.io/builder"
"xorm.io/xorm"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
type deletionMarkerMigrator struct {
migrator.MigrationBase
}
func (m *deletionMarkerMigrator) SQL(dialect migrator.Dialect) string {
return `Find rows in resource_history with value LIKE {"kind":"DeletedMarker"%`
}
func (m *deletionMarkerMigrator) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
logger := log.New("deletion-marker-migrator")
type model struct {
GUID string `xorm:"guid"`
Value string `xorm:"value"`
Group string `xorm:"group"`
Resource string `xorm:"resource"`
PreviousRV string `xorm:"previous_resource_version"`
}
var models []model
logger.Info("finding any deletion markers")
err := sess.Table("resource_history").
Cols("guid", "value", "group", "resource", "previous_resource_version").
Where("action = 3").And("value LIKE ?", `{"kind":"DeletedMarker"%`).
Find(&models)
if err != nil {
return err
}
if len(models) > 0 {
logger.Info("updating markers with a real resource", "count", len(models))
for _, row := range models {
tmp := &model{}
_ = sess.Table("resource_history").Conds().And(builder.Eq(map[string]any{
"group": row.Group,
"resource": row.Resource,
"resource_version": row.PreviousRV,
}))
ok, err := sess.Get(tmp)
if err != nil {
return err
}
if ok && len(tmp.Value) > 1 {
previous := &unstructured.Unstructured{}
err = previous.UnmarshalJSON([]byte(tmp.Value))
if err != nil {
return err
}
marker := &unstructured.Unstructured{}
err = marker.UnmarshalJSON([]byte(row.Value))
if err != nil {
return err
}
// 3. Prepare a new payload
metaMarker, _ := utils.MetaAccessor(marker)
metaPrev, _ := utils.MetaAccessor(previous)
metaPrev.SetDeletionTimestamp(metaMarker.GetDeletionTimestamp())
metaPrev.SetFinalizers(nil)
metaPrev.SetManagedFields(nil)
metaPrev.SetGeneration(utils.DeletedGeneration)
metaPrev.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
ts, _ := metaMarker.GetUpdatedTimestamp()
if ts != nil {
metaPrev.SetUpdatedTimestamp(ts)
}
buff, err := previous.MarshalJSON()
if err != nil {
return err
}
count, err := sess.Table("resource_history").Update(&model{
Value: string(buff),
}, &model{
GUID: row.GUID,
})
if err != nil {
return err
}
if count == 1 {
logger.Info("Updated", "GUID", row.GUID)
} else {
return fmt.Errorf("error updating")
}
} else {
_, err := sess.Table("resource_history").Delete(&model{
GUID: row.GUID,
})
if err != nil {
return err
}
logger.Info("Removed", "GUID", row.GUID)
}
}
}
return nil
}
pkg/storage/unified/sql/db/migrations/resource_mig.go
+2
View File
@@ -149,5 +149,7 @@ func initResourceTables(mg *migrator.Migrator) string {
Name: "folder", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, Default: "''",
}))
mg.AddMigration("Migrate DeletionMarkers to real Resource objects", &deletionMarkerMigrator{})
return marker
}
pkg/storage/unified/sql/migrations.go
-134
View File
@@ -1,134 +0,0 @@
package sql
import (
"context"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/sql/db"
"github.com/grafana/grafana/pkg/storage/unified/sql/dbutil"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
// This runs functions before the server is returned as healthy
func (b *backend) runStartupDataMigrations(ctx context.Context) error {
if b.skipDataMigration {
return nil
}
type migrateRow struct {
GUID string
Marker *unstructured.Unstructured
Group string
Resource string
PreviousRV int64
}
// Migrate DeletedMarker to regular resource
err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
req := &sqlMigrationQueryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
MarkerQuery: `{"kind":"DeletedMarker"%`,
}
// 1. Find rows with the existing deletion marker
rows, err := dbutil.QueryRows(ctx, tx, sqlMigratorGetDeletionMarkers, req)
if err != nil {
return err
}
migrateRows := make([]migrateRow, 0)
for rows.Next() {
item := migrateRow{Marker: &unstructured.Unstructured{}}
err = rows.Scan(&item.GUID, &req.Value, &item.Group, &item.Resource, &item.PreviousRV)
if err != nil {
return err
}
err = item.Marker.UnmarshalJSON([]byte(req.Value))
if err != nil {
return err
}
migrateRows = append(migrateRows, item)
}
err = rows.Close()
if err != nil {
return err
}
for _, item := range migrateRows {
// 2. Load the previous value referenced by that marker
req := &sqlMigrationQueryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
Group: item.Group,
Resource: item.Resource,
RV: item.PreviousRV,
GUID: item.GUID,
}
rows, err = dbutil.QueryRows(ctx, tx, sqlMigratorGetValueFromRV, req)
if err != nil {
return err
}
if rows.Next() {
err = rows.Scan(&req.Value)
if err != nil {
return err
}
}
err = rows.Close()
if err != nil {
return err
}
req.Reset()
if len(req.Value) > 0 {
previous := &unstructured.Unstructured{}
err = previous.UnmarshalJSON([]byte(req.Value))
if err != nil {
return err
}
// 3. Prepare a new payload
metaMarker, _ := utils.MetaAccessor(item.Marker)
metaPrev, _ := utils.MetaAccessor(previous)
metaPrev.SetDeletionTimestamp(metaMarker.GetDeletionTimestamp())
metaPrev.SetFinalizers(nil)
metaPrev.SetManagedFields(nil)
metaPrev.SetGeneration(utils.DeletedGeneration)
metaPrev.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
ts, _ := metaMarker.GetUpdatedTimestamp()
if ts != nil {
metaPrev.SetUpdatedTimestamp(ts)
}
buff, err := previous.MarshalJSON()
if err != nil {
return err
}
req.Value = string(buff)
// 4. Update the SQL row with this new value
b.log.Info("Migrating DeletedMarker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
_, err = dbutil.Exec(ctx, tx, sqlMigratorUpdateValueWithGUID, req)
if err != nil {
return err
}
} else {
// 5. If the previous version is missing, we delete it -- there is nothing to help us restore anyway
b.log.Warn("Removing orphan deletion marker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
_, err = dbutil.Exec(ctx, tx, sqlResourceHistoryDelete, &sqlResourceHistoryDeleteRequest{
SQLTemplate: sqltemplate.New(b.dialect),
GUID: req.GUID,
Namespace: item.Marker.GetNamespace(),
})
if err != nil {
return err
}
}
}
return nil
})
return err
}
pkg/storage/unified/sql/queries.go
-20
View File
@@ -53,10 +53,6 @@ var (
sqlResourceBlobInsert = mustTemplate("resource_blob_insert.sql")
sqlResourceBlobQuery = mustTemplate("resource_blob_query.sql")
sqlMigratorGetDeletionMarkers = mustTemplate("migrator_get_deletion_markers.sql")
sqlMigratorGetValueFromRV = mustTemplate("migrator_get_value_from_rv.sql")
sqlMigratorUpdateValueWithGUID = mustTemplate("migrator_update_value_with_guid.sql")
)
// TxOptions.
@@ -349,19 +345,3 @@ func (r *sqlResourceVersionListRequest) Results() (*groupResourceVersion, error)
x := *r.groupResourceVersion
return &x, nil
}
// This holds all the variables used in migration queries
type sqlMigrationQueryRequest struct {
sqltemplate.SQLTemplate
MarkerQuery string //
Group string
Resource string
RV int64
GUID string
Value string
}
func (r sqlMigrationQueryRequest) Validate() error {
return nil // TODO
}
pkg/storage/unified/sql/queries_test.go
-30
View File
@@ -385,35 +385,5 @@ func TestUnifiedStorageQueries(t *testing.T) {
},
},
},
sqlMigratorGetDeletionMarkers: {
{
Name: "list",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
MarkerQuery: `{"kind":"DeletedMarker"%`,
},
},
},
sqlMigratorGetValueFromRV: {
{
Name: "get",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Group: "ggg",
Resource: "rrr",
RV: 1234,
},
},
},
sqlMigratorUpdateValueWithGUID: {
{
Name: "update",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
GUID: "ggggg",
Value: "{new value}",
},
},
},
}})
}
pkg/storage/unified/sql/testdata/mysql--migrator_get_deletion_markers-list.sql
Vendored
-9
View File
@@ -1,9 +0,0 @@
SELECT
`guid`,
`value`,
`group`,
`resource`,
`previous_resource_version`
FROM `resource_history`
WHERE `action` = 3
AND `value` LIKE '{"kind":"DeletedMarker"%';
pkg/storage/unified/sql/testdata/mysql--migrator_get_value_from_rv-get.sql
Vendored
-5
View File
@@ -1,5 +0,0 @@
SELECT `value`
FROM `resource_history`
WHERE `group` = 'ggg'
AND `resource` = 'rrr'
AND `resource_version` = 1234;
pkg/storage/unified/sql/testdata/mysql--migrator_update_value_with_guid-update.sql
Vendored
-4
View File
@@ -1,4 +0,0 @@
UPDATE `resource_history`
SET `value` = '{new value}'
WHERE `guid` = 'ggggg'
;
pkg/storage/unified/sql/testdata/postgres--migrator_get_deletion_markers-list.sql
Vendored
-9
View File
@@ -1,9 +0,0 @@
SELECT
"guid",
"value",
"group",
"resource",
"previous_resource_version"
FROM "resource_history"
WHERE "action" = 3
AND "value" LIKE '{"kind":"DeletedMarker"%';
pkg/storage/unified/sql/testdata/postgres--migrator_get_value_from_rv-get.sql
Vendored
-5
View File
@@ -1,5 +0,0 @@
SELECT "value"
FROM "resource_history"
WHERE "group" = 'ggg'
AND "resource" = 'rrr'
AND "resource_version" = 1234;
pkg/storage/unified/sql/testdata/postgres--migrator_update_value_with_guid-update.sql
Vendored
-4
View File
@@ -1,4 +0,0 @@
UPDATE "resource_history"
SET "value" = '{new value}'
WHERE "guid" = 'ggggg'
;
pkg/storage/unified/sql/testdata/sqlite--migrator_get_deletion_markers-list.sql
Vendored
-9
View File
@@ -1,9 +0,0 @@
SELECT
"guid",
"value",
"group",
"resource",
"previous_resource_version"
FROM "resource_history"
WHERE "action" = 3
AND "value" LIKE '{"kind":"DeletedMarker"%';
pkg/storage/unified/sql/testdata/sqlite--migrator_get_value_from_rv-get.sql
Vendored
-5
View File
@@ -1,5 +0,0 @@
SELECT "value"
FROM "resource_history"
WHERE "group" = 'ggg'
AND "resource" = 'rrr'
AND "resource_version" = 1234;
pkg/storage/unified/sql/testdata/sqlite--migrator_update_value_with_guid-update.sql
Vendored
-4
View File
@@ -1,4 +0,0 @@
UPDATE "resource_history"
SET "value" = '{new value}'
WHERE "guid" = 'ggggg'
;