[release-12.3.1] Chore: Run annotation data migration in batches (#115136)

* Chore: Run annotation data migration in batches (#113589)

* run annotation data migration in batches

* how could i miss it

* run in the background, starting from newest annotations

* update tests

* optionally pass batch size via env

(cherry picked from commit 95e65e2588)

* update mysql query

---------

Co-authored-by: Serge Zaitsev <serge.zaitsev@grafana.com>

pkg/services/annotations/annotationsimpl/xorm_store.go

@@ -105,10 +105,16 @@ func triggerAlwaysOnMigrations(cfg *setting.Cfg, l log.Logger, db db.DB) {
 		l.Debug("skipped dashboard UID startup migration")
 		return
 	}
-	err := migrations.RunDashboardUIDMigrations(db.GetEngine().NewSession(), db.GetEngine().DriverName())
-	if err != nil {
-		l.Error("failed to populate dashboard_uid for annotations", "error", err)
-	}
+	// Run migration in a background goroutine to avoid blocking service startup
+	go func() {
+		l.Info("Starting annotation dashboard_uid migration in background")
+		err := migrations.RunDashboardUIDMigrations(db.GetEngine().NewSession(), db.GetEngine().DriverName(), l)
+		if err != nil {
+			l.Error("failed to populate dashboard_uid for annotations", "error", err)
+		} else {
+			l.Info("Annotation dashboard_uid migration completed successfully")
+		}
+	}()
 }
 
 func (r *xormRepositoryImpl) Type() string {

pkg/services/annotations/annotationsimpl/xorm_store_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
@@ -768,24 +769,28 @@ func TestIntegrationAnnotationsAlwaysOnMigrations(t *testing.T) {
 		require.NotNil(t, store)
 		assert.Equal(t, "sql", store.Type())
 
+		// Wait for the async migration to complete (runs in background goroutine)
 		var result struct {
 			DashboardUID *string `xorm:"dashboard_uid"`
 		}
-		err = sql.WithDbSession(context.Background(), func(sess *db.Session) error {
-			has, err := sess.Table("annotation").
-				Where("id = ?", annotation.ID).
-				Get(&result)
+		require.Eventually(t, func() bool {
+			err = sql.WithDbSession(context.Background(), func(sess *db.Session) error {
+				has, err := sess.Table("annotation").
+					Where("id = ?", annotation.ID).
+					Get(&result)
+				if err != nil {
+					return err
+				}
+				if !has {
+					return fmt.Errorf("annotation not found")
+				}
+				return nil
+			})
 			if err != nil {
-				return err
+				return false
 			}
-			if !has {
-				return fmt.Errorf("annotation not found")
-			}
-			return nil
-		})
-		require.NoError(t, err)
-		require.NotNil(t, result.DashboardUID, "dashboard_uid should not be NULL when migration runs")
-		assert.Equal(t, "test-run-uid", *result.DashboardUID, "dashboard_uid should be populated when migration runs")
+			return result.DashboardUID != nil && *result.DashboardUID == "test-run-uid"
+		}, 5*time.Second, 100*time.Millisecond, "dashboard_uid should be populated when migration runs")
 	})
 }