feat: enable auto migration based on resource count (#115619)

* feat(unified): migration at startup based on resource count

-- draft

* feat: introduce auto migration enablement for dashboards & folders

* feat: enable auto migration based on threshold

* fix: improve

* fix: pass in the auto migrate per migration definition

* fix: minor

* fix: only use one options

* fix: test

* fix: test

* fix: tests

* fix: simplify configs

* chore: rename

* fix: add integration test

* fix: add integration test

* fix: integration tests

* chore: add comments

* fix: address comment

* fix: address comments

* fix: test and auto migration flow

* fix: test

---------

Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>

pkg/setting/setting.go

@@ -637,6 +637,8 @@ type UnifiedStorageConfig struct {
 	// EnableMigration indicates whether migration is enabled for the resource.
 	// If not set, will use the default from MigratedUnifiedResources.
 	EnableMigration bool
+	// AutoMigrationThreshold is the threshold below which a resource is automatically migrated.
+	AutoMigrationThreshold int
 }
 
 type InstallPlugin struct {

pkg/setting/setting_unified_storage.go

@@ -8,6 +8,10 @@ import (
 	"github.com/grafana/grafana/pkg/util/osutil"
 )
 
+// DefaultAutoMigrationThreshold is the default threshold for auto migration switching.
+// If a resource has entries at or below this count, it will be migrated.
+const DefaultAutoMigrationThreshold = 10
+
 const (
 	PlaylistResource  = "playlists.playlist.grafana.app"
 	FolderResource    = "folders.folder.grafana.app"
@@ -21,6 +25,13 @@ var MigratedUnifiedResources = map[string]bool{
 	DashboardResource: false,
 }
 
+// AutoMigratedUnifiedResources maps resources that support auto-migration
+// TODO: remove this before Grafana 13 GA: https://github.com/grafana/search-and-storage-team/issues/613
+var AutoMigratedUnifiedResources = map[string]bool{
+	FolderResource:    true,
+	DashboardResource: true,
+}
+
 // read storage configs from ini file. They look like:
 // [unified_storage.<group>.<resource>]
 // <field> = <value>
@@ -59,6 +70,13 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 			enableMigration = section.Key("enableMigration").MustBool(MigratedUnifiedResources[resourceName])
 		}
 
+		// parse autoMigrationThreshold from resource section
+		autoMigrationThreshold := 0
+		autoMigrate := AutoMigratedUnifiedResources[resourceName]
+		if autoMigrate {
+			autoMigrationThreshold = section.Key("autoMigrationThreshold").MustInt(DefaultAutoMigrationThreshold)
+		}
+
 		storageConfig[resourceName] = UnifiedStorageConfig{
 			DualWriterMode:                       rest.DualWriterMode(dualWriterMode),
 			DualWriterPeriodicDataSyncJobEnabled: dualWriterPeriodicDataSyncJobEnabled,
@@ -66,6 +84,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 			DataSyncerRecordsLimit:               dataSyncerRecordsLimit,
 			DataSyncerInterval:                   dataSyncerInterval,
 			EnableMigration:                      enableMigration,
+			AutoMigrationThreshold:               autoMigrationThreshold,
 		}
 	}
 	cfg.UnifiedStorage = storageConfig
@@ -73,13 +92,13 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 	// Set indexer config for unified storage
 	section := cfg.Raw.Section("unified_storage")
 	cfg.DisableDataMigrations = section.Key("disable_data_migrations").MustBool(false)
-	if !cfg.DisableDataMigrations && cfg.getUnifiedStorageType() == "unified" {
+	if !cfg.DisableDataMigrations && cfg.UnifiedStorageType() == "unified" {
 		// Helper log to find instances running migrations in the future
 		cfg.Logger.Info("Unified migration configs enforced")
 		cfg.enforceMigrationToUnifiedConfigs()
 	} else {
 		// Helper log to find instances disabling migration
-		cfg.Logger.Info("Unified migration configs enforcement disabled", "storage_type", cfg.getUnifiedStorageType(), "disable_data_migrations", cfg.DisableDataMigrations)
+		cfg.Logger.Info("Unified migration configs enforcement disabled", "storage_type", cfg.UnifiedStorageType(), "disable_data_migrations", cfg.DisableDataMigrations)
 	}
 	cfg.EnableSearch = section.Key("enable_search").MustBool(false)
 	cfg.MaxPageSizeBytes = section.Key("max_page_size_bytes").MustInt(0)
@@ -147,14 +166,15 @@ func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
 			DualWriterMode:                      5,
 			DualWriterMigrationDataSyncDisabled: true,
 			EnableMigration:                     true,
+			AutoMigrationThreshold:              resourceCfg.AutoMigrationThreshold,
 		}
 	}
 }
 
-// getUnifiedStorageType returns the configured storage type without creating or mutating keys.
+// UnifiedStorageType returns the configured storage type without creating or mutating keys.
 // Precedence: env > ini > default ("unified").
 // Used to decide unified storage behavior early without side effects.
-func (cfg *Cfg) getUnifiedStorageType() string {
+func (cfg *Cfg) UnifiedStorageType() string {
 	const (
 		grafanaAPIServerSectionName = "grafana-apiserver"
 		storageTypeKeyName          = "storage_type"
@@ -168,3 +188,23 @@ func (cfg *Cfg) getUnifiedStorageType() string {
 	}
 	return defaultStorageType
 }
+
+// UnifiedStorageConfig returns the UnifiedStorageConfig for a resource.
+func (cfg *Cfg) UnifiedStorageConfig(resource string) UnifiedStorageConfig {
+	if cfg.UnifiedStorage == nil {
+		return UnifiedStorageConfig{}
+	}
+	return cfg.UnifiedStorage[resource]
+}
+
+// EnableMode5 enables migration and sets mode 5 for a resource.
+func (cfg *Cfg) EnableMode5(resource string) {
+	if cfg.UnifiedStorage == nil {
+		cfg.UnifiedStorage = make(map[string]UnifiedStorageConfig)
+	}
+	config := cfg.UnifiedStorage[resource]
+	config.DualWriterMode = rest.Mode5
+	config.DualWriterMigrationDataSyncDisabled = true
+	config.EnableMigration = true
+	cfg.UnifiedStorage[resource] = config
+}

pkg/setting/setting_unified_storage_test.go

@@ -43,10 +43,16 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 				}
 				assert.Equal(t, exists, true, migratedResource)
 
+				expectedThreshold := 0
+				if AutoMigratedUnifiedResources[migratedResource] {
+					expectedThreshold = DefaultAutoMigrationThreshold
+				}
+
 				assert.Equal(t, UnifiedStorageConfig{
 					DualWriterMode:                      5,
 					DualWriterMigrationDataSyncDisabled: true,
 					EnableMigration:                     isEnabled,
+					AutoMigrationThreshold:              expectedThreshold,
 				}, resourceCfg, migratedResource)
 			}
 		}
@@ -71,6 +77,7 @@ func TestCfg_setUnifiedStorageConfig(t *testing.T) {
 			DualWriterPeriodicDataSyncJobEnabled: true,
 			DataSyncerRecordsLimit:               1001,
 			DataSyncerInterval:                   time.Minute * 10,
+			AutoMigrationThreshold:               0,
 		})
 
 		validateMigratedResources(false)