public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(unified): in-proc SQLite data migration (#114537)

Browse Source 
* feat: unified storage migrations integration tests

* chore: add comment and adjust db path name

* chore: refactor test cases into interface

* fix: unified SQLite migration with SQLStore migrator

* revert changes to newResourceDBProvider
This commit is contained in:
Rafael Bortolon Paulovic
2025-11-28 13:13:35 +01:00
committed by GitHub
parent 11a27ab870
commit 12c6d7e83f
7 changed files with 286 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/migrations/resource_migration.go
+16
View File
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/util/xorm"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -72,6 +73,17 @@ func (m *ResourceMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) erro
m.log.Info("Starting migration for all organizations", "org_count", len(orgs), "resources", m.resources)
if mg.Dialect.DriverName() == migrator.SQLite {
// reuse transaction in SQLite to avoid "database is locked" errors
tx, err := sess.Tx()
if err != nil {
m.log.Error("Failed to get transaction from session", "error", err)
return fmt.Errorf("failed to get transaction: %w", err)
}
ctx = resource.ContextWithTransaction(ctx, tx.Tx)
m.log.Info("Stored migrator transaction in context for bulk operations (SQLite compatibility)")
}
for _, org := range orgs {
if err := m.migrateOrg(ctx, sess, org); err != nil {
return err
@@ -107,6 +119,10 @@ func (m *ResourceMigration) migrateOrg(ctx context.Context, sess *xorm.Session,
m.log.Error("Migration failed", "org_id", org.ID, "error", err, "duration", time.Since(startTime))
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, err)
}
if response.Error != nil {
m.log.Error("Migration reported error", "org_id", org.ID, "error", response.Error.String(), "duration", time.Since(startTime))
return fmt.Errorf("migration failed for org %d (%s): %w", org.ID, org.Name, fmt.Errorf("migration error: %s", response.Error.Message))
}
// Validate the migration results
if err := m.validateMigration(migrationCtx, sess, response); err != nil {
pkg/storage/unified/migrations/service.go
+10 -2
View File
@@ -85,8 +85,13 @@ func RegisterMigrations(
// Run all registered migrations (blocking)
sec := cfg.Raw.Section("database")
migrationLocking := sec.Key("migration_locking").MustBool(true)
if mg.Dialect.DriverName() == sqlstoremigrator.SQLite {
// disable migration locking for SQLite to avoid "database is locked" errors in the bulk operations
migrationLocking = false
}
if err := mg.RunMigrations(ctx,
sec.Key("migration_locking").MustBool(true),
migrationLocking,
sec.Key("locking_attempt_timeout_sec").MustInt()); err != nil {
return fmt.Errorf("unified storage data migration failed: %w", err)
}
@@ -98,12 +103,14 @@ func RegisterMigrations(
func registerDashboardAndFolderMigration(mg *sqlstoremigrator.Migrator, migrator UnifiedMigrator, client resource.ResourceClient) {
folders := schema.GroupResource{Group: "folder.grafana.app", Resource: "folders"}
dashboards := schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}
driverName := mg.Dialect.DriverName()
folderCountValidator := NewCountValidator(
client,
folders,
"dashboard",
"org_id = ? and is_folder = true",
driverName,
)
dashboardCountValidator := NewCountValidator(
@@ -111,9 +118,10 @@ func registerDashboardAndFolderMigration(mg *sqlstoremigrator.Migrator, migrator
dashboards,
"dashboard",
"org_id = ? and is_folder = false",
driverName,
)
folderTreeValidator := NewFolderTreeValidator(client, folders)
folderTreeValidator := NewFolderTreeValidator(client, folders, driverName)
dashboardsAndFolders := NewResourceMigration(
migrator,
pkg/storage/unified/migrations/validator.go
+77 -22
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/util/xorm"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -57,6 +58,7 @@ type CountValidator struct {
resource schema.GroupResource
table string
whereClause string
driverName string
}
func NewCountValidator(
@@ -64,6 +66,7 @@ func NewCountValidator(
resource schema.GroupResource,
table string,
whereClause string,
driverName string,
) Validator {
return &CountValidator{
name: "CountValidator",
@@ -71,6 +74,7 @@ func NewCountValidator(
resource: resource,
table: table,
whereClause: whereClause,
driverName: driverName,
}
}
@@ -120,22 +124,32 @@ func (v *CountValidator) Validate(ctx context.Context, sess *xorm.Session, respo
return fmt.Errorf("failed to count %s: %w", v.table, err)
}
// Get unified storage count using GetStats API
statsResp, err := v.client.GetStats(ctx, &resourcepb.ResourceStatsRequest{
Namespace: summary.Namespace,
Kinds: []string{fmt.Sprintf("%s/%s", summary.Group, summary.Resource)},
})
if err != nil {
return fmt.Errorf("failed to get stats for %s/%s in namespace %s: %w",
summary.Group, summary.Resource, summary.Namespace, err)
}
// Find the count for this specific resource type
var unifiedCount int64
for _, stat := range statsResp.Stats {
if stat.Group == summary.Group && stat.Resource == summary.Resource {
unifiedCount = stat.Count
break
if v.driverName == migrator.SQLite {
unifiedCount, err = sess.Table("resource").
Where("namespace = ? AND `group` = ? AND resource = ?",
summary.Namespace, summary.Group, summary.Resource).
Count()
if err != nil {
return fmt.Errorf("failed to count resource table for %s/%s in namespace %s: %w",
summary.Group, summary.Resource, summary.Namespace, err)
}
} else {
// Get unified storage count using GetStats API
statsResp, err := v.client.GetStats(ctx, &resourcepb.ResourceStatsRequest{
Namespace: summary.Namespace,
Kinds: []string{fmt.Sprintf("%s/%s", summary.Group, summary.Resource)},
})
if err != nil {
return fmt.Errorf("failed to get stats for %s/%s in namespace %s: %w",
summary.Group, summary.Resource, summary.Namespace, err)
}
// Find the count for this specific resource type
for _, stat := range statsResp.Stats {
if stat.Group == summary.Group && stat.Resource == summary.Resource {
unifiedCount = stat.Count
break
}
}
}
@@ -162,19 +176,22 @@ func (v *CountValidator) Validate(ctx context.Context, sess *xorm.Session, respo
}
type FolderTreeValidator struct {
name string
client resourcepb.ResourceIndexClient
resource schema.GroupResource
name string
client resourcepb.ResourceIndexClient
resource schema.GroupResource
driverName string
}
func NewFolderTreeValidator(
client resourcepb.ResourceIndexClient,
resource schema.GroupResource,
driverName string,
) Validator {
return &FolderTreeValidator{
name: "FolderTreeValidator",
client: client,
resource: resource,
name: "FolderTreeValidator",
client: client,
resource: resource,
driverName: driverName,
}
}
@@ -185,6 +202,12 @@ type legacyFolder struct {
Title string `xorm:"title"`
}
type unifiedFolder struct {
GUID string `xorm:"guid"`
Name string `xorm:"name"`
Folder string `xorm:"folder"`
}
func (v *FolderTreeValidator) Name() string {
return v.name
}
@@ -218,7 +241,12 @@ func (v *FolderTreeValidator) Validate(ctx context.Context, sess *xorm.Session,
}
// Build unified storage folder parent map
unifiedParentMap, err := v.buildUnifiedFolderParentMap(ctx, summary.Namespace, log)
var unifiedParentMap map[string]string
if v.driverName == migrator.SQLite {
unifiedParentMap, err = v.buildUnifiedFolderParentMapSQLite(sess, summary.Namespace, log)
} else {
unifiedParentMap, err = v.buildUnifiedFolderParentMap(ctx, summary.Namespace, log)
}
if err != nil {
return fmt.Errorf("failed to build unified folder parent map: %w", err)
}
@@ -348,3 +376,30 @@ func (v *FolderTreeValidator) buildUnifiedFolderParentMap(ctx context.Context, n
return parentMap, nil
}
func (v *FolderTreeValidator) buildUnifiedFolderParentMapSQLite(sess *xorm.Session, namespace string, log log.Logger) (map[string]string, error) {
var folders []unifiedFolder
err := sess.Table("resource").
Cols("guid", "name", "folder").
Where("namespace = ? AND resource = ?", namespace, "folder").
Find(&folders)
if err != nil {
return nil, fmt.Errorf("failed to query unified folders: %w", err)
}
parentMap := make(map[string]string)
for _, folder := range folders {
parentMap[folder.Name] = folder.Folder
}
if len(parentMap) == 0 {
log.Debug("No unified folders found for namespace", "namespace", namespace)
return make(map[string]string), nil
}
log.Debug("Built unified folder parent map",
"folder_count", len(parentMap),
"namespace", namespace)
return parentMap, nil
}