feat(dashboard): Org-aware cache for schema migration (#115025)

* fix: use dsIndexProvider cache on migrations

* chore: use same comment as before

* feat: org-aware TTL cache for schemaversion migration and warmup for single tenant

* chore: use LRU cache

* chore: change DefaultCacheTTL to 1 minute

* chore: address copilot reviews

* chore: use expirable cache

* chore: remove unused import

apps/dashboard/go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.7.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/yamux v0.1.2 // indirect
 	github.com/jaegertracing/jaeger-idl v0.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

apps/dashboard/go.sum

@@ -112,6 +112,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA=
 github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8=
 github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns=
 github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE=

apps/dashboard/pkg/migration/conversion/conversion_cache_test.go

@@ -0,0 +1,454 @@
+package conversion
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	dashv0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
+	dashv1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
+	dashv2alpha1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1"
+	dashv2beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2beta1"
+	"github.com/grafana/grafana/apps/dashboard/pkg/migration"
+	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
+	common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// countingDataSourceProvider tracks how many times Index() is called
+type countingDataSourceProvider struct {
+	datasources []schemaversion.DataSourceInfo
+	callCount   atomic.Int64
+}
+
+func newCountingDataSourceProvider(datasources []schemaversion.DataSourceInfo) *countingDataSourceProvider {
+	return &countingDataSourceProvider{
+		datasources: datasources,
+	}
+}
+
+func (p *countingDataSourceProvider) Index(_ context.Context) *schemaversion.DatasourceIndex {
+	p.callCount.Add(1)
+	return schemaversion.NewDatasourceIndex(p.datasources)
+}
+
+func (p *countingDataSourceProvider) getCallCount() int64 {
+	return p.callCount.Load()
+}
+
+// countingLibraryElementProvider tracks how many times GetLibraryElementInfo() is called
+type countingLibraryElementProvider struct {
+	elements  []schemaversion.LibraryElementInfo
+	callCount atomic.Int64
+}
+
+func newCountingLibraryElementProvider(elements []schemaversion.LibraryElementInfo) *countingLibraryElementProvider {
+	return &countingLibraryElementProvider{
+		elements: elements,
+	}
+}
+
+func (p *countingLibraryElementProvider) GetLibraryElementInfo(_ context.Context) []schemaversion.LibraryElementInfo {
+	p.callCount.Add(1)
+	return p.elements
+}
+
+func (p *countingLibraryElementProvider) getCallCount() int64 {
+	return p.callCount.Load()
+}
+
+// createTestV0Dashboard creates a minimal v0 dashboard for testing
+// The dashboard has a datasource with UID only (no type) to force provider lookup
+// and includes library panels to test library element provider caching
+func createTestV0Dashboard(namespace, title string) *dashv0.Dashboard {
+	return &dashv0.Dashboard{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-dashboard",
+			Namespace: namespace,
+		},
+		Spec: common.Unstructured{
+			Object: map[string]interface{}{
+				"title":         title,
+				"schemaVersion": schemaversion.LATEST_VERSION,
+				// Variables with datasource reference that requires lookup
+				"templating": map[string]interface{}{
+					"list": []interface{}{
+						map[string]interface{}{
+							"name":  "query_var",
+							"type":  "query",
+							"query": "label_values(up, job)",
+							// Datasource with UID only - type needs to be looked up
+							"datasource": map[string]interface{}{
+								"uid": "ds1",
+								// type is intentionally omitted to trigger provider lookup
+							},
+						},
+					},
+				},
+				"panels": []interface{}{
+					map[string]interface{}{
+						"id":    1,
+						"title": "Test Panel",
+						"type":  "timeseries",
+						"targets": []interface{}{
+							map[string]interface{}{
+								// Datasource with UID only - type needs to be looked up
+								"datasource": map[string]interface{}{
+									"uid": "ds1",
+								},
+							},
+						},
+					},
+					// Library panel reference - triggers library element provider lookup
+					map[string]interface{}{
+						"id":    2,
+						"title": "Library Panel with Horizontal Repeat",
+						"type":  "library-panel-ref",
+						"gridPos": map[string]interface{}{
+							"h": 8,
+							"w": 12,
+							"x": 0,
+							"y": 8,
+						},
+						"libraryPanel": map[string]interface{}{
+							"uid":  "lib-panel-repeat-h",
+							"name": "Library Panel with Horizontal Repeat",
+						},
+					},
+					// Another library panel reference
+					map[string]interface{}{
+						"id":    3,
+						"title": "Library Panel without Repeat",
+						"type":  "library-panel-ref",
+						"gridPos": map[string]interface{}{
+							"h": 3,
+							"w": 6,
+							"x": 0,
+							"y": 16,
+						},
+						"libraryPanel": map[string]interface{}{
+							"uid":  "lib-panel-no-repeat",
+							"name": "Library Panel without Repeat",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// createTestV1Dashboard creates a minimal v1beta1 dashboard for testing
+// The dashboard has a datasource with UID only (no type) to force provider lookup
+// and includes library panels to test library element provider caching
+func createTestV1Dashboard(namespace, title string) *dashv1.Dashboard {
+	return &dashv1.Dashboard{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-dashboard",
+			Namespace: namespace,
+		},
+		Spec: common.Unstructured{
+			Object: map[string]interface{}{
+				"title":         title,
+				"schemaVersion": schemaversion.LATEST_VERSION,
+				// Variables with datasource reference that requires lookup
+				"templating": map[string]interface{}{
+					"list": []interface{}{
+						map[string]interface{}{
+							"name":  "query_var",
+							"type":  "query",
+							"query": "label_values(up, job)",
+							// Datasource with UID only - type needs to be looked up
+							"datasource": map[string]interface{}{
+								"uid": "ds1",
+								// type is intentionally omitted to trigger provider lookup
+							},
+						},
+					},
+				},
+				"panels": []interface{}{
+					map[string]interface{}{
+						"id":    1,
+						"title": "Test Panel",
+						"type":  "timeseries",
+						"targets": []interface{}{
+							map[string]interface{}{
+								// Datasource with UID only - type needs to be looked up
+								"datasource": map[string]interface{}{
+									"uid": "ds1",
+								},
+							},
+						},
+					},
+					// Library panel reference - triggers library element provider lookup
+					map[string]interface{}{
+						"id":    2,
+						"title": "Library Panel with Vertical Repeat",
+						"type":  "library-panel-ref",
+						"gridPos": map[string]interface{}{
+							"h": 4,
+							"w": 6,
+							"x": 0,
+							"y": 8,
+						},
+						"libraryPanel": map[string]interface{}{
+							"uid":  "lib-panel-repeat-v",
+							"name": "Library Panel with Vertical Repeat",
+						},
+					},
+					// Another library panel reference
+					map[string]interface{}{
+						"id":    3,
+						"title": "Library Panel without Repeat",
+						"type":  "library-panel-ref",
+						"gridPos": map[string]interface{}{
+							"h": 3,
+							"w": 6,
+							"x": 6,
+							"y": 8,
+						},
+						"libraryPanel": map[string]interface{}{
+							"uid":  "lib-panel-no-repeat",
+							"name": "Library Panel without Repeat",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// TestConversionCaching_V0_to_V2alpha1 verifies caching works when converting V0 to V2alpha1
+func TestConversionCaching_V0_to_V2alpha1(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-h", Name: "Library Panel with Horizontal Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, time.Minute)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, time.Minute)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	// Convert multiple dashboards in the same namespace
+	numDashboards := 5
+	namespace := "default"
+
+	for i := 0; i < numDashboards; i++ {
+		source := createTestV0Dashboard(namespace, "Dashboard "+string(rune('A'+i)))
+		target := &dashv2alpha1.Dashboard{}
+
+		err := Convert_V0_to_V2alpha1(source, target, nil, cachedDS, cachedLE)
+		require.NoError(t, err, "conversion %d should succeed", i)
+		require.NotNil(t, target.Spec)
+	}
+
+	// With caching, the underlying datasource provider should only be called once per namespace
+	// The test dashboard has datasources without type that require lookup
+	assert.Equal(t, int64(1), underlyingDS.getCallCount(),
+		"datasource provider should be called only once for %d conversions in same namespace", numDashboards)
+	// Library element provider should also be called only once per namespace due to caching
+	assert.Equal(t, int64(1), underlyingLE.getCallCount(),
+		"library element provider should be called only once for %d conversions in same namespace", numDashboards)
+}
+
+// TestConversionCaching_V0_to_V2beta1 verifies caching works when converting V0 to V2beta1
+func TestConversionCaching_V0_to_V2beta1(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-h", Name: "Library Panel with Horizontal Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, time.Minute)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, time.Minute)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	numDashboards := 5
+	namespace := "default"
+
+	for i := 0; i < numDashboards; i++ {
+		source := createTestV0Dashboard(namespace, "Dashboard "+string(rune('A'+i)))
+		target := &dashv2beta1.Dashboard{}
+
+		err := Convert_V0_to_V2beta1(source, target, nil, cachedDS, cachedLE)
+		require.NoError(t, err, "conversion %d should succeed", i)
+		require.NotNil(t, target.Spec)
+	}
+
+	assert.Equal(t, int64(1), underlyingDS.getCallCount(),
+		"datasource provider should be called only once for %d conversions in same namespace", numDashboards)
+	assert.Equal(t, int64(1), underlyingLE.getCallCount(),
+		"library element provider should be called only once for %d conversions in same namespace", numDashboards)
+}
+
+// TestConversionCaching_V1beta1_to_V2alpha1 verifies caching works when converting V1beta1 to V2alpha1
+func TestConversionCaching_V1beta1_to_V2alpha1(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-v", Name: "Library Panel with Vertical Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, time.Minute)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, time.Minute)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	numDashboards := 5
+	namespace := "default"
+
+	for i := 0; i < numDashboards; i++ {
+		source := createTestV1Dashboard(namespace, "Dashboard "+string(rune('A'+i)))
+		target := &dashv2alpha1.Dashboard{}
+
+		err := Convert_V1beta1_to_V2alpha1(source, target, nil, cachedDS, cachedLE)
+		require.NoError(t, err, "conversion %d should succeed", i)
+		require.NotNil(t, target.Spec)
+	}
+
+	assert.Equal(t, int64(1), underlyingDS.getCallCount(),
+		"datasource provider should be called only once for %d conversions in same namespace", numDashboards)
+	assert.Equal(t, int64(1), underlyingLE.getCallCount(),
+		"library element provider should be called only once for %d conversions in same namespace", numDashboards)
+}
+
+// TestConversionCaching_V1beta1_to_V2beta1 verifies caching works when converting V1beta1 to V2beta1
+func TestConversionCaching_V1beta1_to_V2beta1(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-v", Name: "Library Panel with Vertical Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, time.Minute)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, time.Minute)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	numDashboards := 5
+	namespace := "default"
+
+	for i := 0; i < numDashboards; i++ {
+		source := createTestV1Dashboard(namespace, "Dashboard "+string(rune('A'+i)))
+		target := &dashv2beta1.Dashboard{}
+
+		err := Convert_V1beta1_to_V2beta1(source, target, nil, cachedDS, cachedLE)
+		require.NoError(t, err, "conversion %d should succeed", i)
+		require.NotNil(t, target.Spec)
+	}
+
+	assert.Equal(t, int64(1), underlyingDS.getCallCount(),
+		"datasource provider should be called only once for %d conversions in same namespace", numDashboards)
+	assert.Equal(t, int64(1), underlyingLE.getCallCount(),
+		"library element provider should be called only once for %d conversions in same namespace", numDashboards)
+}
+
+// TestConversionCaching_MultipleNamespaces verifies that different namespaces get separate cache entries
+func TestConversionCaching_MultipleNamespaces(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-h", Name: "Library Panel with Horizontal Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, time.Minute)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, time.Minute)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	namespaces := []string{"default", "org-2", "org-3"}
+	numDashboardsPerNs := 3
+
+	for _, ns := range namespaces {
+		for i := 0; i < numDashboardsPerNs; i++ {
+			source := createTestV0Dashboard(ns, "Dashboard "+string(rune('A'+i)))
+			target := &dashv2alpha1.Dashboard{}
+
+			err := Convert_V0_to_V2alpha1(source, target, nil, cachedDS, cachedLE)
+			require.NoError(t, err, "conversion for namespace %s should succeed", ns)
+		}
+	}
+
+	// With caching, each namespace should result in one call to the underlying provider
+	expectedCalls := int64(len(namespaces))
+	assert.Equal(t, expectedCalls, underlyingDS.getCallCount(),
+		"datasource provider should be called once per namespace (%d namespaces)", len(namespaces))
+	assert.Equal(t, expectedCalls, underlyingLE.getCallCount(),
+		"library element provider should be called once per namespace (%d namespaces)", len(namespaces))
+}
+
+// TestConversionCaching_CacheDisabled verifies that TTL=0 disables caching
+func TestConversionCaching_CacheDisabled(t *testing.T) {
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+	elements := []schemaversion.LibraryElementInfo{
+		{UID: "lib-panel-repeat-h", Name: "Library Panel with Horizontal Repeat", Type: "timeseries"},
+		{UID: "lib-panel-no-repeat", Name: "Library Panel without Repeat", Type: "graph"},
+	}
+
+	underlyingDS := newCountingDataSourceProvider(datasources)
+	underlyingLE := newCountingLibraryElementProvider(elements)
+
+	// TTL of 0 should disable caching - the wrapper returns the underlying provider directly
+	cachedDS := schemaversion.WrapIndexProviderWithCache(underlyingDS, 0)
+	cachedLE := schemaversion.WrapLibraryElementProviderWithCache(underlyingLE, 0)
+
+	migration.ResetForTesting()
+	migration.Initialize(cachedDS, cachedLE, migration.DefaultCacheTTL)
+
+	numDashboards := 3
+	namespace := "default"
+
+	for i := 0; i < numDashboards; i++ {
+		source := createTestV0Dashboard(namespace, "Dashboard "+string(rune('A'+i)))
+		target := &dashv2alpha1.Dashboard{}
+
+		err := Convert_V0_to_V2alpha1(source, target, nil, cachedDS, cachedLE)
+		require.NoError(t, err, "conversion %d should succeed", i)
+	}
+
+	// Without caching, each conversion calls the underlying provider multiple times
+	// (once for each datasource lookup needed - variables and panels)
+	// The key check is that the count is GREATER than 1 per conversion (no caching benefit)
+	assert.Greater(t, underlyingDS.getCallCount(), int64(numDashboards),
+		"with cache disabled, conversions should call datasource provider multiple times")
+	// Library element provider is also called for each conversion without caching
+	assert.GreaterOrEqual(t, underlyingLE.getCallCount(), int64(numDashboards),
+		"with cache disabled, conversions should call library element provider multiple times")
+}

apps/dashboard/pkg/migration/conversion/conversion_data_loss_detection_test.go

@@ -829,7 +829,7 @@ func TestDataLossDetectionOnAllInputFiles(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := testutil.NewDataSourceProvider(testutil.StandardTestConfig)
 	leProvider := testutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()

apps/dashboard/pkg/migration/conversion/conversion_test.go

@@ -35,7 +35,7 @@ func TestConversionMatrixExist(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	versions := []metav1.Object{
 		&dashv0.Dashboard{Spec: common.Unstructured{Object: map[string]any{"title": "dashboardV0"}}},
@@ -89,7 +89,7 @@ func TestDashboardConversionToAllVersions(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
@@ -309,7 +309,7 @@ func TestMigratedDashboardsConversion(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
@@ -428,7 +428,7 @@ func setupTestConversionScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	scheme := runtime.NewScheme()
 	err := RegisterConversions(scheme, dsProvider, leProvider)
@@ -527,7 +527,7 @@ func TestConversionMetrics(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -694,7 +694,7 @@ func TestConversionMetricsWrapper(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -864,7 +864,7 @@ func TestSchemaVersionExtraction(t *testing.T) {
 			dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 			// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 			leProvider := migrationtestutil.NewTestLibraryElementProvider()
-			migration.Initialize(dsProvider, leProvider)
+			migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 			// Create a test registry for metrics
 			registry := prometheus.NewRegistry()
@@ -910,7 +910,7 @@ func TestConversionLogging(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -1003,7 +1003,7 @@ func TestConversionLogLevels(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("log levels and structured fields verification", func(t *testing.T) {
 		// Create test wrapper to verify logging behavior
@@ -1076,7 +1076,7 @@ func TestConversionLoggingFields(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	// Use TestLibraryElementProvider for tests that need library panel models with repeat options
 	leProvider := migrationtestutil.NewTestLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("verify all log fields are present", func(t *testing.T) {
 		// Test that the conversion wrapper includes all expected structured fields

apps/dashboard/pkg/migration/conversion/v0_test.go

@@ -20,7 +20,7 @@ func TestV0ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	tests := []struct {
 		name            string
@@ -132,7 +132,7 @@ func TestV0ConversionErrorPropagation(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("ConvertDashboard_V0_to_V1beta1 returns error on migration failure", func(t *testing.T) {
 		source := &dashv0.Dashboard{
@@ -206,7 +206,7 @@ func TestV0ConversionSuccessPaths(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("Convert_V0_to_V1beta1 success path returns nil", func(t *testing.T) {
 		source := &dashv0.Dashboard{
@@ -275,7 +275,7 @@ func TestV0ConversionSecondStepErrors(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("Convert_V0_to_V2alpha1 sets status on first step error", func(t *testing.T) {
 		// Create a dashboard that will fail v0->v1beta1 conversion

apps/dashboard/pkg/migration/conversion/v1_test.go

@@ -19,7 +19,7 @@ func TestV1ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("Convert_V1beta1_to_V2alpha1 sets status on conversion error", func(t *testing.T) {
 		// Create a dashboard that will cause conversion to fail

apps/dashboard/pkg/migration/conversion/v1beta1_to_v2alpha1_test.go

@@ -19,7 +19,7 @@ func TestV1beta1ToV2alpha1(t *testing.T) {
 	// Initialize the migrator with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()

apps/dashboard/pkg/migration/conversion/v2_test.go

@@ -18,7 +18,7 @@ func TestV2alpha1ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with test data source and library element providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("Convert_V2alpha1_to_V1beta1 sets status on conversion", func(t *testing.T) {
 		// Create a dashboard for conversion
@@ -90,7 +90,7 @@ func TestV2beta1ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with test data source and library element providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	t.Run("Convert_V2beta1_to_V1beta1 sets status on first step failure", func(t *testing.T) {
 		// Create a dashboard that might cause conversion to fail on first step (v2beta1 -> v2alpha1)

apps/dashboard/pkg/migration/conversion/v2alpha1_to_v1beta1_test.go

@@ -282,7 +282,7 @@ func TestV2alpha1ToV1beta1LayoutErrors(t *testing.T) {
 	// Initialize the migrator with test data source and library element providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
@@ -498,7 +498,7 @@ func TestV2alpha1ToV1beta1BasicFields(t *testing.T) {
 	// Initialize the migrator with test data source and library element providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()

apps/dashboard/pkg/migration/conversion/v2alpha1_to_v2beta1_test.go

@@ -18,7 +18,7 @@ func TestV2alpha1ToV2beta1(t *testing.T) {
 	// Initialize the migrator with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()

apps/dashboard/pkg/migration/conversion/v2beta1_to_v2alpha1_test.go

@@ -24,7 +24,7 @@ func TestV2beta1ToV2alpha1RoundTrip(t *testing.T) {
 	// Initialize the migrator with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
@@ -107,7 +107,7 @@ func TestV2beta1ToV2alpha1FromOutputFiles(t *testing.T) {
 	// Initialize the migrator with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
@@ -193,7 +193,7 @@ func TestV2beta1ToV2alpha1(t *testing.T) {
 	// Initialize the migrator with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	migration.Initialize(dsProvider, leProvider)
+	migration.Initialize(dsProvider, leProvider, migration.DefaultCacheTTL)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()

apps/dashboard/pkg/migration/migrate.go

@@ -4,13 +4,19 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
+	"github.com/grafana/authlib/types"
+	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
 )
 
+// DefaultCacheTTL is the default TTL for the datasource and library element caches.
+const DefaultCacheTTL = time.Minute
+
 // Initialize provides the migrator singleton with required dependencies and builds the map of migrations.
-func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
-	migratorInstance.init(dsIndexProvider, leIndexProvider)
+func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider, cacheTTL time.Duration) {
+	migratorInstance.init(dsIndexProvider, leIndexProvider, cacheTTL)
 }
 
 // GetDataSourceIndexProvider returns the datasource index provider instance that was initialized.
@@ -38,6 +44,34 @@ func ResetForTesting() {
 	initOnce = sync.Once{}
 }
 
+// PreloadCache preloads the datasource and library element caches for the given namespaces.
+func PreloadCache(ctx context.Context, nsInfos []types.NamespaceInfo) {
+	// Wait for initialization to complete
+	<-migratorInstance.ready
+
+	// Try to preload datasource cache
+	if preloadable, ok := migratorInstance.dsIndexProvider.(schemaversion.PreloadableCache); ok {
+		preloadable.Preload(ctx, nsInfos)
+	}
+
+	// Try to preload library element cache
+	if preloadable, ok := migratorInstance.leIndexProvider.(schemaversion.PreloadableCache); ok {
+		preloadable.Preload(ctx, nsInfos)
+	}
+}
+
+// PreloadCacheInBackground starts a goroutine that preloads the caches for the given namespaces.
+func PreloadCacheInBackground(nsInfos []types.NamespaceInfo) {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logging.DefaultLogger.Error("panic during cache preloading", "error", r)
+			}
+		}()
+		PreloadCache(context.Background(), nsInfos)
+	}()
+}
+
 // Migrate migrates the given dashboard to the target version.
 // This will block until the migrator is initialized.
 func Migrate(ctx context.Context, dash map[string]interface{}, targetVersion int) error {
@@ -59,14 +93,14 @@ type migrator struct {
 	leIndexProvider schemaversion.LibraryElementIndexProvider
 }
 
-func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
+func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider, cacheTTL time.Duration) {
 	initOnce.Do(func() {
-		// Wrap the provider once with 10s caching for all conversions.
+		// Wrap the provider with org-aware TTL caching for all conversions.
 		// This prevents repeated DB queries across multiple conversion calls while allowing
 		// the cache to refresh periodically, making it suitable for long-lived singleton usage.
-		m.dsIndexProvider = schemaversion.WrapIndexProviderWithCache(dsIndexProvider)
+		m.dsIndexProvider = schemaversion.WrapIndexProviderWithCache(dsIndexProvider, cacheTTL)
 		// Wrap library element provider with caching as well
-		m.leIndexProvider = schemaversion.WrapLibraryElementProviderWithCache(leIndexProvider)
+		m.leIndexProvider = schemaversion.WrapLibraryElementProviderWithCache(leIndexProvider, cacheTTL)
 		m.migrations = schemaversion.GetMigrations(m.dsIndexProvider, m.leIndexProvider)
 		close(m.ready)
 	})

apps/dashboard/pkg/migration/migrate_test.go

@@ -10,10 +10,13 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"k8s.io/apiserver/pkg/endpoints/request"
 
 	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
 	migrationtestutil "github.com/grafana/grafana/apps/dashboard/pkg/migration/testutil"
@@ -31,7 +34,7 @@ func TestMigrate(t *testing.T) {
 	ResetForTesting()
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	Initialize(dsProvider, leProvider)
+	Initialize(dsProvider, leProvider, DefaultCacheTTL)
 
 	t.Run("minimum version check", func(t *testing.T) {
 		err := Migrate(context.Background(), map[string]interface{}{
@@ -49,7 +52,7 @@ func TestMigrateSingleVersion(t *testing.T) {
 	// Use the same datasource provider as the frontend test to ensure consistency
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	Initialize(dsProvider, leProvider)
+	Initialize(dsProvider, leProvider, DefaultCacheTTL)
 
 	runSingleVersionMigrationTests(t, SINGLE_VERSION_OUTPUT_DIR)
 }
@@ -218,7 +221,7 @@ func TestSchemaMigrationMetrics(t *testing.T) {
 	// Initialize migration with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	Initialize(dsProvider, leProvider)
+	Initialize(dsProvider, leProvider, DefaultCacheTTL)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -304,7 +307,7 @@ func TestSchemaMigrationMetrics(t *testing.T) {
 func TestSchemaMigrationLogging(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	Initialize(dsProvider, leProvider)
+	Initialize(dsProvider, leProvider, DefaultCacheTTL)
 
 	tests := []struct {
 		name           string
@@ -423,7 +426,7 @@ func TestMigrateDevDashboards(t *testing.T) {
 	ResetForTesting()
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.DevDashboardConfig)
 	leProvider := migrationtestutil.NewLibraryElementProvider()
-	Initialize(dsProvider, leProvider)
+	Initialize(dsProvider, leProvider, DefaultCacheTTL)
 
 	runDevDashboardMigrationTests(t, schemaversion.LATEST_VERSION, DEV_DASHBOARDS_OUTPUT_DIR)
 }
@@ -449,3 +452,232 @@ func runDevDashboardMigrationTests(t *testing.T, targetVersion int, outputDir st
 		})
 	}
 }
+
+func TestMigrateWithCache(t *testing.T) {
+	// Reset the migration singleton before each test
+	ResetForTesting()
+	datasources := []schemaversion.DataSourceInfo{
+		{UID: "ds-uid-1", Type: "prometheus", Name: "Prometheus", Default: true, APIVersion: "v1"},
+		{UID: "ds-uid-2", Type: "loki", Name: "Loki", Default: false, APIVersion: "v1"},
+		{UID: "ds-uid-3", Type: "prometheus", Name: "Prometheus 2", Default: false, APIVersion: "v1"},
+	}
+
+	// Create a dashboard at schema version 32 for V33 and V36 migration with datasource references
+	dashboard1 := map[string]interface{}{
+		"schemaVersion": 32,
+		"title":         "Test Dashboard 1",
+		"panels": []interface{}{
+			map[string]interface{}{
+				"id":   1,
+				"type": "timeseries",
+				// String datasource that V33 will migrate to object reference
+				"datasource": "Prometheus",
+				"targets": []interface{}{
+					map[string]interface{}{
+						"refId":      "A",
+						"datasource": "Loki",
+					},
+				},
+			},
+		},
+	}
+
+	// Create a dashboard at schema version 35 for testing V36 migration with datasource references in annotations
+	dashboard2 := map[string]interface{}{
+		"schemaVersion": 35,
+		"title":         "Test Dashboard 2",
+		"annotations": map[string]interface{}{
+			"list": []interface{}{
+				map[string]interface{}{
+					"name":       "Test Annotation",
+					"datasource": "Prometheus 2", // String reference that V36 should convert
+					"enable":     true,
+				},
+			},
+		},
+	}
+
+	t.Run("with datasources", func(t *testing.T) {
+		ResetForTesting()
+		dsProvider := newCountingProvider(datasources)
+		leProvider := newCountingLibraryProvider(nil)
+		// Initialize the migration system with our counting providers
+		Initialize(dsProvider, leProvider, DefaultCacheTTL)
+		// Verify initial call count is zero
+		assert.Equal(t, dsProvider.GetCallCount(), int64(0))
+		// Create a context with namespace (required for caching)
+		ctx := request.WithNamespace(context.Background(), "default")
+
+		// First migration - should invoke the provider once to build the cache
+		dash1 := deepCopyDashboard(dashboard1)
+		err := Migrate(ctx, dash1, schemaversion.LATEST_VERSION)
+		require.NoError(t, err)
+		assert.Equal(t, int64(1), dsProvider.GetCallCount())
+
+		// Verify datasource conversion from string to object reference
+		panels := dash1["panels"].([]interface{})
+		panel := panels[0].(map[string]interface{})
+		panelDS, ok := panel["datasource"].(map[string]interface{})
+		require.True(t, ok, "panel datasource should be converted to object")
+		assert.Equal(t, "ds-uid-1", panelDS["uid"])
+		assert.Equal(t, "prometheus", panelDS["type"])
+
+		// Verify target datasource conversion
+		targets := panel["targets"].([]interface{})
+		target := targets[0].(map[string]interface{})
+		targetDS, ok := target["datasource"].(map[string]interface{})
+		require.True(t, ok, "target datasource should be converted to object")
+		assert.Equal(t, "ds-uid-2", targetDS["uid"])
+		assert.Equal(t, "loki", targetDS["type"])
+
+		// Migration with V35 dashboard - should use the cached index from first migration
+		dash2 := deepCopyDashboard(dashboard2)
+		err = Migrate(ctx, dash2, schemaversion.LATEST_VERSION)
+		require.NoError(t, err, "second migration should succeed")
+		assert.Equal(t, int64(1), dsProvider.GetCallCount())
+
+		// Verify the annotation datasource was converted to object reference
+		annotations := dash2["annotations"].(map[string]interface{})
+		list := annotations["list"].([]interface{})
+		var testAnnotation map[string]interface{}
+		for _, a := range list {
+			ann := a.(map[string]interface{})
+			if ann["name"] == "Test Annotation" {
+				testAnnotation = ann
+				break
+			}
+		}
+		require.NotNil(t, testAnnotation, "Test Annotation should exist")
+		annotationDS, ok := testAnnotation["datasource"].(map[string]interface{})
+		require.True(t, ok, "annotation datasource should be converted to object")
+		assert.Equal(t, "ds-uid-3", annotationDS["uid"])
+		assert.Equal(t, "prometheus", annotationDS["type"])
+	})
+
+	// tests that cache isolates data per namespace
+	t.Run("with multiple orgs", func(t *testing.T) {
+		// Reset the migration singleton
+		ResetForTesting()
+		dsProvider := newCountingProvider(datasources)
+		leProvider := newCountingLibraryProvider(nil)
+		Initialize(dsProvider, leProvider, DefaultCacheTTL)
+		// Create contexts for different orgs with proper namespace format (org-ID)
+		ctx1 := request.WithNamespace(context.Background(), "default")  // org 1
+		ctx2 := request.WithNamespace(context.Background(), "stacks-2") // stack 2
+
+		// Migrate for org 1
+		err := Migrate(ctx1, deepCopyDashboard(dashboard1), schemaversion.LATEST_VERSION)
+		require.NoError(t, err)
+		callsAfterOrg1 := dsProvider.GetCallCount()
+
+		// Migrate for org 2 - should build separate cache
+		err = Migrate(ctx2, deepCopyDashboard(dashboard2), schemaversion.LATEST_VERSION)
+		require.NoError(t, err)
+		callsAfterOrg2 := dsProvider.GetCallCount()
+
+		assert.Greater(t, callsAfterOrg2, callsAfterOrg1,
+			"org 2 migration should have called provider (separate cache)")
+
+		// Migrate again for org 1 - should use cache
+		err = Migrate(ctx1, deepCopyDashboard(dashboard1), schemaversion.LATEST_VERSION)
+		require.NoError(t, err)
+		callsAfterOrg1Again := dsProvider.GetCallCount()
+		assert.Equal(t, callsAfterOrg2, callsAfterOrg1Again,
+			"second org 1 migration should use cache")
+
+		// Migrate again for org 2 - should use cache
+		err = Migrate(ctx2, deepCopyDashboard(dashboard1), schemaversion.LATEST_VERSION)
+		require.NoError(t, err)
+		callsAfterOrg2Again := dsProvider.GetCallCount()
+		assert.Equal(t, callsAfterOrg2, callsAfterOrg2Again,
+			"second org 2 migration should use cache")
+	})
+}
+
+// countingProvider wraps a datasource provider and counts calls to Index()
+type countingProvider struct {
+	datasources []schemaversion.DataSourceInfo
+	callCount   atomic.Int64
+}
+
+func newCountingProvider(datasources []schemaversion.DataSourceInfo) *countingProvider {
+	return &countingProvider{
+		datasources: datasources,
+	}
+}
+
+func (p *countingProvider) Index(_ context.Context) *schemaversion.DatasourceIndex {
+	p.callCount.Add(1)
+	return schemaversion.NewDatasourceIndex(p.datasources)
+}
+
+func (p *countingProvider) GetCallCount() int64 {
+	return p.callCount.Load()
+}
+
+// countingLibraryProvider wraps a library element provider and counts calls
+type countingLibraryProvider struct {
+	elements  []schemaversion.LibraryElementInfo
+	callCount atomic.Int64
+}
+
+func newCountingLibraryProvider(elements []schemaversion.LibraryElementInfo) *countingLibraryProvider {
+	return &countingLibraryProvider{
+		elements: elements,
+	}
+}
+
+func (p *countingLibraryProvider) GetLibraryElementInfo(_ context.Context) []schemaversion.LibraryElementInfo {
+	p.callCount.Add(1)
+	return p.elements
+}
+
+func (p *countingLibraryProvider) GetCallCount() int64 {
+	return p.callCount.Load()
+}
+
+// deepCopyDashboard creates a deep copy of a dashboard map
+func deepCopyDashboard(dash map[string]interface{}) map[string]interface{} {
+	cpy := make(map[string]interface{})
+	for k, v := range dash {
+		switch val := v.(type) {
+		case []interface{}:
+			cpy[k] = deepCopySlice(val)
+		case map[string]interface{}:
+			cpy[k] = deepCopyMapForCache(val)
+		default:
+			cpy[k] = v
+		}
+	}
+	return cpy
+}
+
+func deepCopySlice(s []interface{}) []interface{} {
+	cpy := make([]interface{}, len(s))
+	for i, v := range s {
+		switch val := v.(type) {
+		case []interface{}:
+			cpy[i] = deepCopySlice(val)
+		case map[string]interface{}:
+			cpy[i] = deepCopyMapForCache(val)
+		default:
+			cpy[i] = v
+		}
+	}
+	return cpy
+}
+
+func deepCopyMapForCache(m map[string]interface{}) map[string]interface{} {
+	cpy := make(map[string]interface{})
+	for k, v := range m {
+		switch val := v.(type) {
+		case []interface{}:
+			cpy[k] = deepCopySlice(val)
+		case map[string]interface{}:
+			cpy[k] = deepCopyMapForCache(val)
+		default:
+			cpy[k] = v
+		}
+	}
+	return cpy
+}

apps/dashboard/pkg/migration/schemaversion/cache.go

@@ -0,0 +1,104 @@
+package schemaversion
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/grafana/authlib/types"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/hashicorp/golang-lru/v2/expirable"
+	k8srequest "k8s.io/apiserver/pkg/endpoints/request"
+
+	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
+)
+
+const defaultCacheSize = 1000
+
+// CacheProvider is a generic cache interface for schema version providers.
+type CacheProvider[T any] interface {
+	// Get returns the cached value if it's still valid, otherwise calls fetch and caches the result.
+	Get(ctx context.Context) T
+}
+
+// PreloadableCache is an interface for providers that support preloading the cache.
+type PreloadableCache interface {
+	// Preload loads data into the cache for the given namespaces.
+	Preload(ctx context.Context, nsInfos []types.NamespaceInfo)
+}
+
+// cachedProvider is a thread-safe TTL cache that wraps any fetch function.
+type cachedProvider[T any] struct {
+	fetch    func(context.Context) T
+	cache    *expirable.LRU[string, T] // LRU cache: namespace to cache entry
+	inFlight sync.Map                  // map[string]*sync.Mutex - per-namespace fetch locks
+	logger   log.Logger
+}
+
+// newCachedProvider creates a new cachedProvider.
+// The fetch function should be able to handle context with different namespaces.
+// A non-positive size turns LRU mechanism off (cache of unlimited size).
+// A non-positive cacheTTL disables TTL expiration.
+func newCachedProvider[T any](fetch func(context.Context) T, size int, cacheTTL time.Duration, logger log.Logger) *cachedProvider[T] {
+	cacheProvider := &cachedProvider[T]{
+		fetch:  fetch,
+		logger: logger,
+	}
+	cacheProvider.cache = expirable.NewLRU(size, func(key string, value T) {
+		cacheProvider.inFlight.Delete(key)
+	}, cacheTTL)
+	return cacheProvider
+}
+
+// Get returns the cached value if it's still valid, otherwise calls fetch and caches the result.
+func (p *cachedProvider[T]) Get(ctx context.Context) T {
+	// Get namespace info from ctx
+	nsInfo, err := request.NamespaceInfoFrom(ctx, true)
+	if err != nil {
+		// No namespace, fall back to direct fetch call without caching
+		p.logger.Warn("Unable to get namespace info from context, skipping cache", "error", err)
+		return p.fetch(ctx)
+	}
+
+	namespace := nsInfo.Value
+	// Fast path: check if cache is still valid
+	if entry, ok := p.cache.Get(namespace); ok {
+		return entry
+	}
+
+	// Get or create a per-namespace lock for this fetch operation
+	// This ensures only one fetch happens per namespace at a time
+	lockInterface, _ := p.inFlight.LoadOrStore(namespace, &sync.Mutex{})
+	nsMutex := lockInterface.(*sync.Mutex)
+
+	// Lock this specific namespace - other namespaces can still proceed
+	nsMutex.Lock()
+	defer nsMutex.Unlock()
+
+	// Double-check: another goroutine might have already fetched while we waited
+	if entry, ok := p.cache.Get(namespace); ok {
+		return entry
+	}
+
+	// Fetch outside the main lock - only this namespace is blocked
+	p.logger.Debug("cache miss or expired, fetching new value", "namespace", namespace)
+	value := p.fetch(ctx)
+
+	// Update the cache for this namespace
+	p.cache.Add(namespace, value)
+
+	return value
+}
+
+// Preload loads data into the cache for the given namespaces.
+func (p *cachedProvider[T]) Preload(ctx context.Context, nsInfos []types.NamespaceInfo) {
+	// Build the cache using a context with the namespace
+	p.logger.Info("preloading cache", "nsInfos", len(nsInfos))
+	startedAt := time.Now()
+	defer func() {
+		p.logger.Info("finished preloading cache", "nsInfos", len(nsInfos), "elapsed", time.Since(startedAt))
+	}()
+	for _, nsInfo := range nsInfos {
+		p.cache.Add(nsInfo.Value, p.fetch(k8srequest.WithNamespace(ctx, nsInfo.Value)))
+	}
+}

apps/dashboard/pkg/migration/schemaversion/cache_test.go

@@ -0,0 +1,478 @@
+package schemaversion
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	authlib "github.com/grafana/authlib/types"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// testProvider tracks how many times get() is called
+type testProvider struct {
+	testData  any
+	callCount atomic.Int64
+}
+
+func newTestProvider(testData any) *testProvider {
+	return &testProvider{
+		testData: testData,
+	}
+}
+
+func (p *testProvider) get(_ context.Context) any {
+	p.callCount.Add(1)
+	return p.testData
+}
+
+func (p *testProvider) getCallCount() int64 {
+	return p.callCount.Load()
+}
+
+func TestCachedProvider_CacheHit(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+		{UID: "ds2", Type: "loki", Name: "Loki"},
+	}
+
+	underlying := newTestProvider(datasources)
+	// Test newCachedProvider directly instead of the wrapper
+	cached := newCachedProvider(underlying.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Use "default" namespace (org 1) - this is the standard Grafana namespace format
+	ctx := request.WithNamespace(context.Background(), "default")
+
+	// First call should hit the underlying provider
+	idx1 := cached.Get(ctx)
+	require.NotNil(t, idx1)
+	assert.Equal(t, int64(1), underlying.getCallCount(), "first call should invoke underlying provider")
+
+	// Second call should use cache
+	idx2 := cached.Get(ctx)
+	require.NotNil(t, idx2)
+	assert.Equal(t, int64(1), underlying.getCallCount(), "second call should use cache, not invoke underlying provider")
+
+	// Both should return the same data
+	assert.Equal(t, idx1, idx2)
+}
+
+func TestCachedProvider_NamespaceIsolation(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+
+	underlying := newTestProvider(datasources)
+	cached := newCachedProvider(underlying.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Use "default" (org 1) and "org-2" (org 2) - standard Grafana namespace formats
+	ctx1 := request.WithNamespace(context.Background(), "default")
+	ctx2 := request.WithNamespace(context.Background(), "org-2")
+
+	// First call for org 1
+	idx1 := cached.Get(ctx1)
+	require.NotNil(t, idx1)
+	assert.Equal(t, int64(1), underlying.getCallCount(), "first org-1 call should invoke underlying provider")
+
+	// Call for org 2 should also invoke underlying provider (different namespace)
+	idx2 := cached.Get(ctx2)
+	require.NotNil(t, idx2)
+	assert.Equal(t, int64(2), underlying.getCallCount(), "org-2 call should invoke underlying provider (separate cache)")
+
+	// Second call for org 1 should use cache
+	idx3 := cached.Get(ctx1)
+	require.NotNil(t, idx3)
+	assert.Equal(t, int64(2), underlying.getCallCount(), "second org-1 call should use cache")
+
+	// Second call for org 2 should use cache
+	idx4 := cached.Get(ctx2)
+	require.NotNil(t, idx4)
+	assert.Equal(t, int64(2), underlying.getCallCount(), "second org-2 call should use cache")
+}
+
+func TestCachedProvider_NoNamespaceFallback(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+
+	underlying := newTestProvider(datasources)
+	cached := newCachedProvider(underlying.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Context without namespace - should fall back to direct provider call
+	ctx := context.Background()
+
+	idx1 := cached.Get(ctx)
+	require.NotNil(t, idx1)
+	assert.Equal(t, int64(1), underlying.getCallCount())
+
+	// Second call without namespace should also invoke underlying (no caching for unknown namespace)
+	idx2 := cached.Get(ctx)
+	require.NotNil(t, idx2)
+	assert.Equal(t, int64(2), underlying.getCallCount(), "without namespace, each call should invoke underlying provider")
+}
+
+func TestCachedProvider_ConcurrentAccess(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+
+	underlying := newTestProvider(datasources)
+	cached := newCachedProvider(underlying.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Use "default" namespace (org 1)
+	ctx := request.WithNamespace(context.Background(), "default")
+
+	var wg sync.WaitGroup
+	numGoroutines := 100
+
+	// Launch many goroutines that all try to access the cache simultaneously
+	for i := 0; i < numGoroutines; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			idx := cached.Get(ctx)
+			require.NotNil(t, idx)
+		}()
+	}
+
+	wg.Wait()
+
+	// Due to double-check locking, only 1 goroutine should have actually built the cache
+	// In practice, there might be a few more due to timing, but it should be much less than numGoroutines
+	callCount := underlying.getCallCount()
+	assert.LessOrEqual(t, callCount, int64(5), "with proper locking, very few goroutines should invoke underlying provider; got %d", callCount)
+}
+
+func TestCachedProvider_ConcurrentNamespaces(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+
+	underlying := newTestProvider(datasources)
+	cached := newCachedProvider(underlying.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	var wg sync.WaitGroup
+	numOrgs := 10
+	callsPerOrg := 20
+
+	// Launch goroutines for multiple namespaces
+	// Use valid namespace formats: "default" for org 1, "org-N" for N > 1
+	namespaces := make([]string, numOrgs)
+	namespaces[0] = "default"
+	for i := 1; i < numOrgs; i++ {
+		namespaces[i] = fmt.Sprintf("org-%d", i+1)
+	}
+
+	for _, ns := range namespaces {
+		ctx := request.WithNamespace(context.Background(), ns)
+		for i := 0; i < callsPerOrg; i++ {
+			wg.Add(1)
+			go func(ctx context.Context) {
+				defer wg.Done()
+				idx := cached.Get(ctx)
+				require.NotNil(t, idx)
+			}(ctx)
+		}
+	}
+
+	wg.Wait()
+
+	// Each org should have at most a few calls (ideally 1, but timing can cause a few more)
+	callCount := underlying.getCallCount()
+	// With 10 orgs, we expect around 10 calls (one per org)
+	assert.LessOrEqual(t, callCount, int64(numOrgs), "expected roughly one call per org, got %d calls for %d orgs", callCount, numOrgs)
+}
+
+// Test that cache returns correct data for each namespace
+func TestCachedProvider_CorrectDataPerNamespace(t *testing.T) {
+	// Provider that returns different data based on namespace
+	underlying := &namespaceAwareProvider{
+		datasourcesByNamespace: map[string][]DataSourceInfo{
+			"default": {{UID: "org1-ds", Type: "prometheus", Name: "Org1 DS", Default: true}},
+			"org-2":   {{UID: "org2-ds", Type: "loki", Name: "Org2 DS", Default: true}},
+		},
+	}
+	cached := newCachedProvider(underlying.Index, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Use valid namespace formats
+	ctx1 := request.WithNamespace(context.Background(), "default")
+	ctx2 := request.WithNamespace(context.Background(), "org-2")
+
+	idx1 := cached.Get(ctx1)
+	idx2 := cached.Get(ctx2)
+
+	assert.Equal(t, "org1-ds", idx1.GetDefault().UID, "org 1 should get org-1 datasources")
+	assert.Equal(t, "org2-ds", idx2.GetDefault().UID, "org 2 should get org-2 datasources")
+
+	// Subsequent calls should still return correct data
+	idx1Again := cached.Get(ctx1)
+	idx2Again := cached.Get(ctx2)
+
+	assert.Equal(t, "org1-ds", idx1Again.GetDefault().UID, "org 1 should still get org-1 datasources from cache")
+	assert.Equal(t, "org2-ds", idx2Again.GetDefault().UID, "org 2 should still get org-2 datasources from cache")
+}
+
+// TestCachedProvider_PreloadMultipleNamespaces verifies preloading multiple namespaces
+func TestCachedProvider_PreloadMultipleNamespaces(t *testing.T) {
+	// Provider that returns different data based on namespace
+	underlying := &namespaceAwareProvider{
+		datasourcesByNamespace: map[string][]DataSourceInfo{
+			"default": {{UID: "org1-ds", Type: "prometheus", Name: "Org1 DS", Default: true}},
+			"org-2":   {{UID: "org2-ds", Type: "loki", Name: "Org2 DS", Default: true}},
+			"org-3":   {{UID: "org3-ds", Type: "tempo", Name: "Org3 DS", Default: true}},
+		},
+	}
+	cached := newCachedProvider(underlying.Index, defaultCacheSize, time.Minute, log.New("test"))
+
+	// Preload multiple namespaces
+	nsInfos := []authlib.NamespaceInfo{
+		createNamespaceInfo(1, 0, "default"),
+		createNamespaceInfo(2, 0, "org-2"),
+		createNamespaceInfo(3, 0, "org-3"),
+	}
+	cached.Preload(context.Background(), nsInfos)
+
+	// After preload, the underlying provider should have been called once per namespace
+	assert.Equal(t, 3, underlying.callCount, "preload should call underlying provider once per namespace")
+
+	// Access all namespaces - should use preloaded data and get correct data per namespace
+	expectedUIDs := map[string]string{
+		"default": "org1-ds",
+		"org-2":   "org2-ds",
+		"org-3":   "org3-ds",
+	}
+
+	for _, ns := range []string{"default", "org-2", "org-3"} {
+		ctx := request.WithNamespace(context.Background(), ns)
+		idx := cached.Get(ctx)
+		require.NotNil(t, idx, "index for namespace %s should not be nil", ns)
+		assert.Equal(t, expectedUIDs[ns], idx.GetDefault().UID, "namespace %s should get correct datasource", ns)
+	}
+
+	// The underlying provider should still have been called only 3 times (from preload)
+	assert.Equal(t, 3, underlying.callCount,
+		"access after preload should use cached data for all namespaces")
+}
+
+// namespaceAwareProvider returns different datasources based on namespace
+type namespaceAwareProvider struct {
+	datasourcesByNamespace map[string][]DataSourceInfo
+	callCount              int
+}
+
+func (p *namespaceAwareProvider) Index(ctx context.Context) *DatasourceIndex {
+	p.callCount++
+	ns := request.NamespaceValue(ctx)
+	if ds, ok := p.datasourcesByNamespace[ns]; ok {
+		return NewDatasourceIndex(ds)
+	}
+	return NewDatasourceIndex(nil)
+}
+
+// createNamespaceInfo creates a NamespaceInfo for testing
+func createNamespaceInfo(orgID, stackID int64, value string) authlib.NamespaceInfo {
+	return authlib.NamespaceInfo{
+		OrgID:   orgID,
+		StackID: stackID,
+		Value:   value,
+	}
+}
+
+// Test DatasourceIndex functionality
+func TestDatasourceIndex_Lookup(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds-uid-1", Type: "prometheus", Name: "Prometheus DS", Default: true, APIVersion: "v1"},
+		{UID: "ds-uid-2", Type: "loki", Name: "Loki DS", Default: false, APIVersion: "v1"},
+	}
+	idx := NewDatasourceIndex(datasources)
+
+	t.Run("lookup by name", func(t *testing.T) {
+		ds := idx.Lookup("Prometheus DS")
+		require.NotNil(t, ds)
+		assert.Equal(t, "ds-uid-1", ds.UID)
+	})
+
+	t.Run("lookup by UID", func(t *testing.T) {
+		ds := idx.Lookup("ds-uid-2")
+		require.NotNil(t, ds)
+		assert.Equal(t, "Loki DS", ds.Name)
+	})
+
+	t.Run("lookup unknown returns nil", func(t *testing.T) {
+		ds := idx.Lookup("unknown")
+		assert.Nil(t, ds)
+	})
+
+	t.Run("get default", func(t *testing.T) {
+		ds := idx.GetDefault()
+		require.NotNil(t, ds)
+		assert.Equal(t, "ds-uid-1", ds.UID)
+	})
+
+	t.Run("lookup by UID directly", func(t *testing.T) {
+		ds := idx.LookupByUID("ds-uid-1")
+		require.NotNil(t, ds)
+		assert.Equal(t, "Prometheus DS", ds.Name)
+	})
+
+	t.Run("lookup by name directly", func(t *testing.T) {
+		ds := idx.LookupByName("Loki DS")
+		require.NotNil(t, ds)
+		assert.Equal(t, "ds-uid-2", ds.UID)
+	})
+}
+
+func TestDatasourceIndex_EmptyIndex(t *testing.T) {
+	idx := NewDatasourceIndex(nil)
+
+	assert.Nil(t, idx.GetDefault())
+	assert.Nil(t, idx.Lookup("anything"))
+	assert.Nil(t, idx.LookupByUID("anything"))
+	assert.Nil(t, idx.LookupByName("anything"))
+}
+
+// TestCachedProvider_TTLExpiration verifies that cache expires after TTL
+func TestCachedProvider_TTLExpiration(t *testing.T) {
+	datasources := []DataSourceInfo{
+		{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+	}
+
+	underlying := newTestProvider(datasources)
+	// Use a very short TTL for testing
+	shortTTL := 50 * time.Millisecond
+	cached := newCachedProvider(underlying.get, defaultCacheSize, shortTTL, log.New("test"))
+
+	ctx := request.WithNamespace(context.Background(), "default")
+
+	// First call - should call underlying provider
+	idx1 := cached.Get(ctx)
+	require.NotNil(t, idx1)
+	assert.Equal(t, int64(1), underlying.getCallCount(), "first call should invoke underlying provider")
+
+	// Second call immediately - should use cache
+	idx2 := cached.Get(ctx)
+	require.NotNil(t, idx2)
+	assert.Equal(t, int64(1), underlying.getCallCount(), "second call should use cache")
+
+	// Wait for TTL to expire
+	time.Sleep(shortTTL + 20*time.Millisecond)
+
+	// Third call after TTL - should call underlying provider again
+	idx3 := cached.Get(ctx)
+	require.NotNil(t, idx3)
+	assert.Equal(t, int64(2), underlying.getCallCount(),
+		"after TTL expiration, underlying provider should be called again")
+}
+
+// TestCachedProvider_ParallelNamespacesFetch verifies that different namespaces can fetch in parallel
+func TestCachedProvider_ParallelNamespacesFetch(t *testing.T) {
+	// Create a blocking provider that tracks concurrent executions
+	provider := &blockingProvider{
+		blockDuration: 100 * time.Millisecond,
+		datasources: []DataSourceInfo{
+			{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+		},
+	}
+	cached := newCachedProvider(provider.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	numNamespaces := 5
+	var wg sync.WaitGroup
+
+	// Launch fetches for different namespaces simultaneously
+	startTime := time.Now()
+	for i := 0; i < numNamespaces; i++ {
+		wg.Add(1)
+		namespace := fmt.Sprintf("org-%d", i+1)
+		go func(ns string) {
+			defer wg.Done()
+			ctx := request.WithNamespace(context.Background(), ns)
+			idx := cached.Get(ctx)
+			require.NotNil(t, idx)
+		}(namespace)
+	}
+	wg.Wait()
+	elapsed := time.Since(startTime)
+
+	// Verify that all namespaces were called
+	assert.Equal(t, int64(numNamespaces), provider.callCount.Load())
+
+	// Verify max concurrent executions shows parallelism
+	maxConcurrent := provider.maxConcurrent.Load()
+	assert.Equal(t, int64(numNamespaces), maxConcurrent)
+
+	// If all namespaces had to wait sequentially, it would take numNamespaces * blockDuration
+	// With parallelism, it should be much faster (close to just blockDuration)
+	sequentialTime := time.Duration(numNamespaces) * provider.blockDuration
+	assert.Less(t, elapsed, sequentialTime)
+}
+
+// TestCachedProvider_SameNamespaceSerialFetch verifies that the same namespace doesn't fetch concurrently
+func TestCachedProvider_SameNamespaceSerialFetch(t *testing.T) {
+	// Create a blocking provider that tracks concurrent executions
+	provider := &blockingProvider{
+		blockDuration: 100 * time.Millisecond,
+		datasources: []DataSourceInfo{
+			{UID: "ds1", Type: "prometheus", Name: "Prometheus", Default: true},
+		},
+	}
+	cached := newCachedProvider(provider.get, defaultCacheSize, time.Minute, log.New("test"))
+
+	numGoroutines := 10
+	var wg sync.WaitGroup
+
+	// Launch multiple fetches for the SAME namespace simultaneously
+	ctx := request.WithNamespace(context.Background(), "default")
+	for i := 0; i < numGoroutines; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			idx := cached.Get(ctx)
+			require.NotNil(t, idx)
+		}()
+	}
+	wg.Wait()
+
+	// Max concurrent should be 1 since all goroutines are for the same namespace
+	maxConcurrent := provider.maxConcurrent.Load()
+	assert.Equal(t, int64(1), maxConcurrent)
+}
+
+// blockingProvider is a test provider that simulates slow fetch operations
+// and tracks concurrent executions
+type blockingProvider struct {
+	blockDuration time.Duration
+	datasources   []DataSourceInfo
+	callCount     atomic.Int64
+	currentActive atomic.Int64
+	maxConcurrent atomic.Int64
+}
+
+func (p *blockingProvider) get(_ context.Context) any {
+	p.callCount.Add(1)
+
+	// Track concurrent executions
+	current := p.currentActive.Add(1)
+
+	// Update max concurrent if this is a new peak
+	for {
+		maxVal := p.maxConcurrent.Load()
+		if current <= maxVal {
+			break
+		}
+		if p.maxConcurrent.CompareAndSwap(maxVal, current) {
+			break
+		}
+	}
+
+	// Simulate slow operation
+	time.Sleep(p.blockDuration)
+
+	p.currentActive.Add(-1)
+	return p.datasources
+}

apps/dashboard/pkg/migration/schemaversion/datasource_utils.go

@@ -2,8 +2,9 @@ package schemaversion
 
 import (
 	"context"
-	"sync"
 	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
 )
 
 // Shared utility functions for datasource migrations across different schema versions.
@@ -11,65 +12,41 @@ import (
 // string names/UIDs to structured reference objects with uid, type, and apiVersion.
 
 // cachedIndexProvider wraps a DataSourceIndexProvider with time-based caching.
-// This prevents multiple DB queries and index builds during operations that may call
-// provider.Index() multiple times (e.g., dashboard conversions with many datasource lookups).
-// The cache expires after 10 seconds, allowing it to be used as a long-lived singleton
-// while still refreshing periodically.
-//
-// Thread-safe: Uses sync.RWMutex to guarantee safe concurrent access.
 type cachedIndexProvider struct {
-	provider DataSourceIndexProvider
-	mu       sync.RWMutex
-	index    *DatasourceIndex
-	cachedAt time.Time
-	cacheTTL time.Duration
+	*cachedProvider[*DatasourceIndex]
 }
 
-// Index returns the cached index if it's still valid (< 10s old), otherwise rebuilds it.
-// Uses RWMutex for efficient concurrent reads when cache is valid.
+// Index returns the cached index if it's still valid (< TTL old), otherwise rebuilds it.
 func (p *cachedIndexProvider) Index(ctx context.Context) *DatasourceIndex {
-	// Fast path: check if cache is still valid using read lock
-	p.mu.RLock()
-	if p.index != nil && time.Since(p.cachedAt) < p.cacheTTL {
-		idx := p.index
-		p.mu.RUnlock()
-		return idx
-	}
-	p.mu.RUnlock()
-
-	// Slow path: cache expired or not yet built, acquire write lock
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	// Double-check: another goroutine might have refreshed the cache
-	// while we were waiting for the write lock
-	if p.index != nil && time.Since(p.cachedAt) < p.cacheTTL {
-		return p.index
-	}
-
-	// Rebuild the cache
-	p.index = p.provider.Index(ctx)
-	p.cachedAt = time.Now()
-	return p.index
+	return p.Get(ctx)
 }
 
-// WrapIndexProviderWithCache wraps a provider to cache the index with a 10-second TTL.
-// Useful for conversions or migrations that may call provider.Index() multiple times.
-// The cache expires after 10 seconds, making it suitable for use as a long-lived singleton
-// at the top level of dependency injection while still refreshing periodically.
-//
-// Example usage in dashboard conversion:
-//
-//	cachedDsIndexProvider := schemaversion.WrapIndexProviderWithCache(dsIndexProvider)
-//	// Now all calls to cachedDsIndexProvider.Index(ctx) return the same cached index
-//	// for up to 10 seconds before refreshing
-func WrapIndexProviderWithCache(provider DataSourceIndexProvider) DataSourceIndexProvider {
-	if provider == nil {
-		return nil
+// cachedLibraryElementProvider wraps a LibraryElementIndexProvider with time-based caching.
+type cachedLibraryElementProvider struct {
+	*cachedProvider[[]LibraryElementInfo]
+}
+
+func (p *cachedLibraryElementProvider) GetLibraryElementInfo(ctx context.Context) []LibraryElementInfo {
+	return p.Get(ctx)
+}
+
+// WrapIndexProviderWithCache wraps a DataSourceIndexProvider to cache indexes with a configurable TTL.
+func WrapIndexProviderWithCache(provider DataSourceIndexProvider, cacheTTL time.Duration) DataSourceIndexProvider {
+	if provider == nil || cacheTTL <= 0 {
+		return provider
 	}
 	return &cachedIndexProvider{
-		provider: provider,
-		cacheTTL: 10 * time.Second,
+		newCachedProvider[*DatasourceIndex](provider.Index, defaultCacheSize, cacheTTL, log.New("schemaversion.dsindexprovider")),
+	}
+}
+
+// WrapLibraryElementProviderWithCache wraps a LibraryElementIndexProvider to cache library elements with a configurable TTL.
+func WrapLibraryElementProviderWithCache(provider LibraryElementIndexProvider, cacheTTL time.Duration) LibraryElementIndexProvider {
+	if provider == nil || cacheTTL <= 0 {
+		return provider
+	}
+	return &cachedLibraryElementProvider{
+		newCachedProvider[[]LibraryElementInfo](provider.GetLibraryElementInfo, defaultCacheSize, cacheTTL, log.New("schemaversion.leindexprovider")),
 	}
 }
 
@@ -216,60 +193,3 @@ func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool,
 
 	return nil
 }
-
-// cachedLibraryElementProvider wraps a LibraryElementIndexProvider with time-based caching.
-// This prevents multiple DB queries during operations that may call GetLibraryElementInfo()
-// multiple times (e.g., dashboard conversions with many library panel lookups).
-// The cache expires after 10 seconds, allowing it to be used as a long-lived singleton
-// while still refreshing periodically.
-//
-// Thread-safe: Uses sync.RWMutex to guarantee safe concurrent access.
-type cachedLibraryElementProvider struct {
-	provider LibraryElementIndexProvider
-	mu       sync.RWMutex
-	elements []LibraryElementInfo
-	cachedAt time.Time
-	cacheTTL time.Duration
-}
-
-// GetLibraryElementInfo returns the cached library elements if they're still valid (< 10s old), otherwise rebuilds the cache.
-// Uses RWMutex for efficient concurrent reads when cache is valid.
-func (p *cachedLibraryElementProvider) GetLibraryElementInfo(ctx context.Context) []LibraryElementInfo {
-	// Fast path: check if cache is still valid using read lock
-	p.mu.RLock()
-	if p.elements != nil && time.Since(p.cachedAt) < p.cacheTTL {
-		elements := p.elements
-		p.mu.RUnlock()
-		return elements
-	}
-	p.mu.RUnlock()
-
-	// Slow path: cache expired or not yet built, acquire write lock
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	// Double-check: another goroutine might have refreshed the cache
-	// while we were waiting for the write lock
-	if p.elements != nil && time.Since(p.cachedAt) < p.cacheTTL {
-		return p.elements
-	}
-
-	// Rebuild the cache
-	p.elements = p.provider.GetLibraryElementInfo(ctx)
-	p.cachedAt = time.Now()
-	return p.elements
-}
-
-// WrapLibraryElementProviderWithCache wraps a provider to cache library elements with a 10-second TTL.
-// Useful for conversions or migrations that may call GetLibraryElementInfo() multiple times.
-// The cache expires after 10 seconds, making it suitable for use as a long-lived singleton
-// at the top level of dependency injection while still refreshing periodically.
-func WrapLibraryElementProviderWithCache(provider LibraryElementIndexProvider) LibraryElementIndexProvider {
-	if provider == nil {
-		return nil
-	}
-	return &cachedLibraryElementProvider{
-		provider: provider,
-		cacheTTL: 10 * time.Second,
-	}
-}

pkg/registry/apis/dashboard/mutation_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestDashboardAPIBuilder_Mutate(t *testing.T) {
-	migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig), testutil.NewLibraryElementProvider())
+	migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig), testutil.NewLibraryElementProvider(), migration.DefaultCacheTTL)
 	tests := []struct {
 		name                string
 		inputObj            runtime.Object

pkg/registry/apis/dashboard/register.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"maps"
+	"strconv"
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -195,13 +196,30 @@ func RegisterAPIService(
 		datasourceService: datasourceService,
 	}, &libraryElementIndexProvider{
 		libraryElementService: libraryPanels,
-	})
+	}, cfg.DashboardSchemaMigrationCacheTTL)
+
+	// For single-tenant deployments (indicated by StackID), preload the cache in the background
+	if cfg.StackID != "" {
+		// Single namespace for cloud stack
+		stackID, err := strconv.ParseInt(cfg.StackID, 10, 64)
+		if err == nil {
+			var nsInfo authlib.NamespaceInfo
+			nsInfo, err = authlib.ParseNamespace(authlib.CloudNamespaceFormatter(stackID))
+			if err == nil {
+				migration.PreloadCacheInBackground([]authlib.NamespaceInfo{nsInfo})
+			}
+		}
+		if err != nil {
+			logging.DefaultLogger.Error("failed to parse namespace for cache preloading", "stackId", cfg.StackID, "err", err)
+		}
+	}
+
 	apiregistration.RegisterAPI(builder)
 	return builder
 }
 
 func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, libraryElementProvider schemaversion.LibraryElementIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder {
-	migration.Initialize(datasourceProvider, libraryElementProvider)
+	migration.Initialize(datasourceProvider, libraryElementProvider, migration.DefaultCacheTTL)
 	return &DashboardsAPIBuilder{
 		minRefreshInterval:     "10s",
 		accessClient:           ac,

pkg/setting/setting.go

@@ -247,11 +247,12 @@ type Cfg struct {
 	MetricsGrafanaEnvironmentInfo    map[string]string
 
 	// Dashboards
-	DashboardVersionsToKeep     int
-	MinRefreshInterval          string
-	DefaultHomeDashboardPath    string
-	DashboardPerformanceMetrics []string
-	PanelSeriesLimit            int
+	DashboardVersionsToKeep          int
+	MinRefreshInterval               string
+	DefaultHomeDashboardPath         string
+	DashboardPerformanceMetrics      []string
+	PanelSeriesLimit                 int
+	DashboardSchemaMigrationCacheTTL time.Duration
 
 	// Auth
 	LoginCookieName               string
@@ -1237,6 +1238,7 @@ func (cfg *Cfg) parseINIFile(iniFile *ini.File) error {
 	cfg.DefaultHomeDashboardPath = dashboards.Key("default_home_dashboard_path").MustString("")
 	cfg.DashboardPerformanceMetrics = util.SplitString(dashboards.Key("dashboard_performance_metrics").MustString(""))
 	cfg.PanelSeriesLimit = dashboards.Key("panel_series_limit").MustInt(0)
+	cfg.DashboardSchemaMigrationCacheTTL = dashboards.Key("schema_migration_cache_ttl").MustDuration(time.Minute)
 
 	if err := readUserSettings(iniFile, cfg); err != nil {
 		return err