Unified: Run resource data migrations at startup (#114857)

* chore: uncomment unified migration

* chore: adapt and fix tests

* chore: dynamically bump max conns if needed during migration

* chore: copilot suggestions

* chore: pass ctx in RegisterMigration

* chore: make playlists opt-out and dashboards opt-in

* chore: adjust dashboard test

* chore: disable enable log in test

* chore: address review comments

- do not use pointer config
- add migration registry

* chore: more consistent naming

* chore: fix playlist discovery test

pkg/setting/setting.go

@@ -632,6 +632,9 @@ type UnifiedStorageConfig struct {
 	DataSyncerInterval time.Duration
 	// DataSyncerRecordsLimit defines how many records will be processed at max during a sync invocation.
 	DataSyncerRecordsLimit int
+	// EnableMigration indicates whether migration is enabled for the resource.
+	// If not set, will use the default from MigratedUnifiedResources.
+	EnableMigration bool
 }
 
 type InstallPlugin struct {

pkg/setting/setting_unified_storage.go

@@ -8,10 +8,17 @@ import (
 	"github.com/grafana/grafana/pkg/util/osutil"
 )
 
-var MigratedUnifiedResources = []string{
-	"playlists.playlist.grafana.app",
-	"folders.folder.grafana.app",
-	"dashboards.dashboard.grafana.app",
+const (
+	PlaylistResource  = "playlists.playlist.grafana.app"
+	FolderResource    = "folders.folder.grafana.app"
+	DashboardResource = "dashboards.dashboard.grafana.app"
+)
+
+// MigratedUnifiedResources maps resources to a boolean indicating if migration is enabled by default
+var MigratedUnifiedResources = map[string]bool{
+	PlaylistResource:  true, // enabled by default
+	FolderResource:    false,
+	DashboardResource: false,
 }
 
 // read storage configs from ini file. They look like:
@@ -46,12 +53,19 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 		// parse dataSyncerInterval from resource section
 		dataSyncerInterval := section.Key("dataSyncerInterval").MustDuration(time.Hour)
 
+		// parse EnableMigration from resource section
+		enableMigration := MigratedUnifiedResources[resourceName]
+		if section.HasKey("enableMigration") {
+			enableMigration = section.Key("enableMigration").MustBool(MigratedUnifiedResources[resourceName])
+		}
+
 		storageConfig[resourceName] = UnifiedStorageConfig{
 			DualWriterMode:                       rest.DualWriterMode(dualWriterMode),
 			DualWriterPeriodicDataSyncJobEnabled: dualWriterPeriodicDataSyncJobEnabled,
 			DualWriterMigrationDataSyncDisabled:  dualWriterMigrationDataSyncDisabled,
 			DataSyncerRecordsLimit:               dataSyncerRecordsLimit,
 			DataSyncerInterval:                   dataSyncerInterval,
+			EnableMigration:                      enableMigration,
 		}
 	}
 	cfg.UnifiedStorage = storageConfig
@@ -61,8 +75,8 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 	cfg.DisableDataMigrations = section.Key("disable_data_migrations").MustBool(false)
 	if !cfg.DisableDataMigrations && cfg.getUnifiedStorageType() == "unified" {
 		// Helper log to find instances running migrations in the future
-		cfg.Logger.Info("Unified migration configs not yet enforced")
-		// cfg.enforceMigrationToUnifiedConfigs() // TODO: uncomment when ready for release
+		cfg.Logger.Info("Unified migration configs enforced")
+		cfg.enforceMigrationToUnifiedConfigs()
 	} else {
 		// Helper log to find instances disabling migration
 		cfg.Logger.Info("Unified migration configs enforcement disabled", "storage_type", cfg.getUnifiedStorageType(), "disable_data_migrations", cfg.DisableDataMigrations)
@@ -107,7 +121,6 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 	cfg.MinFileIndexBuildVersion = section.Key("min_file_index_build_version").MustString("")
 }
 
-// nolint:unused
 // enforceMigrationToUnifiedConfigs enforces configurations required to run migrated resources in mode 5
 // All migrated resources in MigratedUnifiedResources are set to mode 5 and unified search is enabled
 func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
@@ -118,14 +131,22 @@ func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
 		section.Key("enable_search").SetValue("true")
 		cfg.EnableSearch = true
 	}
-	for _, resource := range MigratedUnifiedResources {
-		cfg.Logger.Info("Enforcing mode 5 for resource in unified storage", "resource", resource)
-		if oldCfg, ok := cfg.UnifiedStorage[resource]; ok {
-			cfg.Logger.Info("Overriding unified storage config for migrated resource", "resource", resource, "old_config", oldCfg)
+	for resource, enabledByDefault := range MigratedUnifiedResources {
+		resourceCfg, ok := cfg.UnifiedStorage[resource]
+		if ok {
+			if !resourceCfg.EnableMigration {
+				cfg.Logger.Info("Resource migration disabled", "resource", resource)
+				continue
+			}
+			cfg.Logger.Info("Overriding unified storage config for migrated resource", "resource", resource, "old_config", resourceCfg)
+		} else if !enabledByDefault {
+			continue
 		}
+		cfg.Logger.Info("Enforcing mode 5 for resource in unified storage", "resource", resource)
 		cfg.UnifiedStorage[resource] = UnifiedStorageConfig{
 			DualWriterMode:                      5,
 			DualWriterMigrationDataSyncDisabled: true,
+			EnableMigration:                     true,
 		}
 	}
 }

pkg/setting/setting_unified_storage_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -13,31 +14,56 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 		err := cfg.Load(CommandLineArgs{HomePath: "../../", Config: "../../conf/defaults.ini"})
 		assert.NoError(t, err)
 
-		s, err := cfg.Raw.NewSection("unified_storage.playlists.playlist.grafana.app")
-		assert.NoError(t, err)
+		setSectionKey := func(sectionName, key, value string) {
+			section := cfg.Raw.Section(sectionName) // Gets existing or creates new
+			_, err := section.NewKey(key, value)
+			assert.NoError(t, err)
+		}
 
-		_, err = s.NewKey("dualWriterMode", "2")
-		assert.NoError(t, err)
+		setMigratedResourceKey := func(key, value string) {
+			for migratedResource := range MigratedUnifiedResources {
+				setSectionKey("unified_storage."+migratedResource, key, value)
+			}
+		}
 
-		_, err = s.NewKey("dualWriterPeriodicDataSyncJobEnabled", "true")
-		assert.NoError(t, err)
+		validateMigratedResources := func(optIn bool) {
+			for migratedResource, enabled := range MigratedUnifiedResources {
+				resourceCfg, exists := cfg.UnifiedStorage[migratedResource]
 
-		_, err = s.NewKey("dataSyncerRecordsLimit", "1001")
-		assert.NoError(t, err)
+				isEnabled := enabled
+				if optIn {
+					isEnabled = true
+				}
 
-		_, err = s.NewKey("dataSyncerInterval", "10m")
-		assert.NoError(t, err)
+				if !isEnabled {
+					if exists {
+						assert.Equal(t, rest.DualWriterMode(1), resourceCfg.DualWriterMode, migratedResource)
+					}
+					continue
+				}
+				assert.Equal(t, exists, true, migratedResource)
+
+				assert.Equal(t, UnifiedStorageConfig{
+					DualWriterMode:                      5,
+					DualWriterMigrationDataSyncDisabled: true,
+					EnableMigration:                     isEnabled,
+				}, resourceCfg, migratedResource)
+			}
+		}
+
+		setMigratedResourceKey("dualWriterMode", "1") // migrated resources enabled by default will change to 5 in setUnifiedStorageConfig
+
+		setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dualWriterMode", "2")
+		setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dualWriterPeriodicDataSyncJobEnabled", "true")
+		setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dataSyncerRecordsLimit", "1001")
+		setSectionKey("unified_storage.resource.not_migrated.grafana.app", "dataSyncerInterval", "10m")
 
 		// Add unified_storage section for index settings
-		unifiedStorageSection, err := cfg.Raw.NewSection("unified_storage")
-		assert.NoError(t, err)
-
-		_, err = unifiedStorageSection.NewKey("index_min_count", "5")
-		assert.NoError(t, err)
+		setSectionKey("unified_storage", "index_min_count", "5")
 
 		cfg.setUnifiedStorageConfig()
 
-		value, exists := cfg.UnifiedStorage["playlists.playlist.grafana.app"]
+		value, exists := cfg.UnifiedStorage["resource.not_migrated.grafana.app"]
 
 		assert.Equal(t, exists, true)
 		assert.Equal(t, value, UnifiedStorageConfig{
@@ -47,6 +73,14 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 			DataSyncerInterval:                   time.Minute * 10,
 		})
 
+		validateMigratedResources(false)
+
+		setMigratedResourceKey("enableMigration", "true") // will be changed to 5 in setUnifiedStorageConfig
+
+		cfg.setUnifiedStorageConfig()
+
+		validateMigratedResources(true)
+
 		// Test that index settings are correctly parsed
 		assert.Equal(t, 5, cfg.IndexMinCount)
 	})