feat: add library elements to dash service (#114016)

apps/dashboard/pkg/migration/conversion/conversion.go

@@ -11,7 +11,7 @@ import (
 	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
 )
 
-func RegisterConversions(s *runtime.Scheme, dsIndexProvider schemaversion.DataSourceIndexProvider) error {
+func RegisterConversions(s *runtime.Scheme, dsIndexProvider schemaversion.DataSourceIndexProvider, _ schemaversion.LibraryElementIndexProvider) error {
 	// Wrap the provider once with 10s caching for all conversions.
 	// This prevents repeated DB queries across multiple conversion calls while allowing
 	// the cache to refresh periodically, making it suitable for long-lived singleton usage.

apps/dashboard/pkg/migration/conversion/conversion_data_loss_detection_test.go

@@ -828,11 +828,12 @@ func TestWithConversionValidation_DataLoss(t *testing.T) {
 func TestDataLossDetectionOnAllInputFiles(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := testutil.NewDataSourceProvider(testutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := testutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	// Read all files from input directory

apps/dashboard/pkg/migration/conversion/conversion_test.go

@@ -33,7 +33,8 @@ import (
 func TestConversionMatrixExist(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	versions := []metav1.Object{
 		&dashv0.Dashboard{Spec: common.Unstructured{Object: map[string]any{"title": "dashboardV0"}}},
@@ -43,7 +44,7 @@ func TestConversionMatrixExist(t *testing.T) {
 	}
 
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	for idx, in := range versions {
@@ -85,11 +86,12 @@ func TestDeepCopyValid(t *testing.T) {
 func TestDashboardConversionToAllVersions(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	// Read all files from input directory
@@ -244,11 +246,12 @@ func TestDashboardConversionToAllVersions(t *testing.T) {
 func TestMigratedDashboardsConversion(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	// Read all files from migration package's latest_version directory
@@ -378,7 +381,8 @@ func testConversion(t *testing.T, convertedDash metav1.Object, filename, outputD
 func TestConversionMetrics(t *testing.T) {
 	// Initialize migration with test providers
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -386,7 +390,7 @@ func TestConversionMetrics(t *testing.T) {
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	tests := []struct {
@@ -505,7 +509,8 @@ func TestConversionMetrics(t *testing.T) {
 // TestConversionMetricsWrapper tests the withConversionMetrics wrapper function
 func TestConversionMetricsWrapper(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -673,7 +678,8 @@ func TestSchemaVersionExtraction(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			// Test the schema version extraction logic by creating a wrapper and checking the metrics labels
 			dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-			migration.Initialize(dsProvider)
+			leProvider := migrationtestutil.NewLibraryElementProvider()
+			migration.Initialize(dsProvider, leProvider)
 
 			// Create a test registry for metrics
 			registry := prometheus.NewRegistry()
@@ -717,7 +723,8 @@ func TestSchemaVersionExtraction(t *testing.T) {
 // TestConversionLogging tests that conversion-level logging works correctly
 func TestConversionLogging(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -725,7 +732,7 @@ func TestConversionLogging(t *testing.T) {
 
 	// Set up conversion scheme
 	scheme := runtime.NewScheme()
-	err := RegisterConversions(scheme, dsProvider)
+	err := RegisterConversions(scheme, dsProvider, leProvider)
 	require.NoError(t, err)
 
 	tests := []struct {
@@ -808,7 +815,8 @@ func TestConversionLogging(t *testing.T) {
 // TestConversionLogLevels tests that appropriate log levels are used
 func TestConversionLogLevels(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("log levels and structured fields verification", func(t *testing.T) {
 		// Create test wrapper to verify logging behavior
@@ -879,7 +887,8 @@ func TestConversionLogLevels(t *testing.T) {
 // TestConversionLoggingFields tests that all expected fields are included in log messages
 func TestConversionLoggingFields(t *testing.T) {
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("verify all log fields are present", func(t *testing.T) {
 		// Test that the conversion wrapper includes all expected structured fields

apps/dashboard/pkg/migration/conversion/v0_test.go

@@ -19,7 +19,8 @@ import (
 func TestV0ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	tests := []struct {
 		name            string
@@ -130,7 +131,8 @@ func TestV0ConversionErrorHandling(t *testing.T) {
 func TestV0ConversionErrorPropagation(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("ConvertDashboard_V0_to_V1beta1 returns error on migration failure", func(t *testing.T) {
 		source := &dashv0.Dashboard{
@@ -203,7 +205,8 @@ func TestV0ConversionErrorPropagation(t *testing.T) {
 func TestV0ConversionSuccessPaths(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("Convert_V0_to_V1beta1 success path returns nil", func(t *testing.T) {
 		source := &dashv0.Dashboard{
@@ -271,7 +274,8 @@ func TestV0ConversionSuccessPaths(t *testing.T) {
 func TestV0ConversionSecondStepErrors(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("Convert_V0_to_V2alpha1 sets status on first step error", func(t *testing.T) {
 		// Create a dashboard that will fail v0->v1beta1 conversion

apps/dashboard/pkg/migration/conversion/v1_test.go

@@ -18,7 +18,8 @@ import (
 func TestV1ConversionErrorHandling(t *testing.T) {
 	// Initialize the migrator with a test data source provider
 	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
-	migration.Initialize(dsProvider)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	migration.Initialize(dsProvider, leProvider)
 
 	t.Run("Convert_V1beta1_to_V2alpha1 sets status on conversion error", func(t *testing.T) {
 		// Create a dashboard that will cause conversion to fail

apps/dashboard/pkg/migration/migrate.go

@@ -9,8 +9,8 @@ import (
 )
 
 // Initialize provides the migrator singleton with required dependencies and builds the map of migrations.
-func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider) {
+func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
-	migratorInstance.init(dsIndexProvider)
+	migratorInstance.init(dsIndexProvider, leIndexProvider)
 }
 
 // GetDataSourceIndexProvider returns the datasource index provider instance that was initialized.
@@ -20,12 +20,20 @@ func GetDataSourceIndexProvider() schemaversion.DataSourceIndexProvider {
 	return migratorInstance.dsIndexProvider
 }
 
+// GetLibraryElementIndexProvider returns the library element index provider instance that was initialized.
+func GetLibraryElementIndexProvider() schemaversion.LibraryElementIndexProvider {
+	// Wait for initialization to complete
+	<-migratorInstance.ready
+	return migratorInstance.leIndexProvider
+}
+
 // ResetForTesting resets the migrator singleton for testing purposes.
 func ResetForTesting() {
 	migratorInstance = &migrator{
 		migrations:      map[int]schemaversion.SchemaVersionMigrationFunc{},
 		ready:           make(chan struct{}),
 		dsIndexProvider: nil,
+		leIndexProvider: nil,
 	}
 	initOnce = sync.Once{}
 }
@@ -48,12 +56,14 @@ type migrator struct {
 	ready           chan struct{}
 	migrations      map[int]schemaversion.SchemaVersionMigrationFunc
 	dsIndexProvider schemaversion.DataSourceIndexProvider
+	leIndexProvider schemaversion.LibraryElementIndexProvider
 }
 
-func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider) {
+func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
 	initOnce.Do(func() {
 		m.dsIndexProvider = dsIndexProvider
-		m.migrations = schemaversion.GetMigrations(dsIndexProvider)
+		m.leIndexProvider = leIndexProvider
+		m.migrations = schemaversion.GetMigrations(dsIndexProvider, leIndexProvider)
 		close(m.ready)
 	})
 }

apps/dashboard/pkg/migration/migrate_test.go

@@ -29,7 +29,9 @@ const DEV_DASHBOARDS_OUTPUT_DIR = "testdata/dev-dashboards-output"
 func TestMigrate(t *testing.T) {
 	// Reset the migration singleton and use the same datasource provider as the frontend test to ensure consistency
 	ResetForTesting()
-	Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
+	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	Initialize(dsProvider, leProvider)
 
 	t.Run("minimum version check", func(t *testing.T) {
 		err := Migrate(context.Background(), map[string]interface{}{
@@ -45,7 +47,9 @@ func TestMigrate(t *testing.T) {
 
 func TestMigrateSingleVersion(t *testing.T) {
 	// Use the same datasource provider as the frontend test to ensure consistency
-	Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
+	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	Initialize(dsProvider, leProvider)
 
 	runSingleVersionMigrationTests(t, SINGLE_VERSION_OUTPUT_DIR)
 }
@@ -212,7 +216,9 @@ func loadDashboard(t *testing.T, path string) map[string]interface{} {
 // TestSchemaMigrationMetrics tests that schema migration metrics are recorded correctly
 func TestSchemaMigrationMetrics(t *testing.T) {
 	// Initialize migration with test providers
-	Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
+	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	Initialize(dsProvider, leProvider)
 
 	// Create a test registry for metrics
 	registry := prometheus.NewRegistry()
@@ -296,7 +302,9 @@ func TestSchemaMigrationMetrics(t *testing.T) {
 
 // TestSchemaMigrationLogging tests that schema migration logging works correctly
 func TestSchemaMigrationLogging(t *testing.T) {
-	Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
+	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	Initialize(dsProvider, leProvider)
 
 	tests := []struct {
 		name           string
@@ -413,7 +421,9 @@ func TestMigrateDevDashboards(t *testing.T) {
 	// Reset the migration singleton and use the dev dashboard datasource provider
 	// to match the frontend devDashboardDataSources configuration
 	ResetForTesting()
-	Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.DevDashboardConfig))
+	dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.DevDashboardConfig)
+	leProvider := migrationtestutil.NewLibraryElementProvider()
+	Initialize(dsProvider, leProvider)
 
 	runDevDashboardMigrationTests(t, schemaversion.LATEST_VERSION, DEV_DASHBOARDS_OUTPUT_DIR)
 }

apps/dashboard/pkg/migration/schemaversion/migrations.go

@@ -27,12 +27,27 @@ type DataSourceIndexProvider interface {
 	Index(ctx context.Context) *DatasourceIndex
 }
 
+type LibraryElementInfo struct {
+	UID         string
+	Name        string
+	Kind        int64
+	Type        string
+	Description string
+	FolderUID   string
+}
+
+type LibraryElementIndexProvider interface {
+
+	// GetLibraryElementInfo returns library element information for use in migrations.
+	GetLibraryElementInfo(ctx context.Context) []LibraryElementInfo
+}
+
 type PanelPluginInfo struct {
 	ID      string
 	Version string
 }
 
-func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc {
+func GetMigrations(dsIndexProvider DataSourceIndexProvider, _ LibraryElementIndexProvider) map[int]SchemaVersionMigrationFunc {
 	return map[int]SchemaVersionMigrationFunc{
 		2:  V2,
 		3:  V3,

apps/dashboard/pkg/migration/testutil/mocks.go

@@ -6,6 +6,19 @@ import (
 	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
 )
 
+// EmptyLibraryElementProvider provides an empty library element list for tests
+type EmptyLibraryElementProvider struct{}
+
+// NewLibraryElementProvider creates a new empty library element provider for tests
+func NewLibraryElementProvider() *EmptyLibraryElementProvider {
+	return &EmptyLibraryElementProvider{}
+}
+
+// GetLibraryElementInfo returns an empty list for tests
+func (p *EmptyLibraryElementProvider) GetLibraryElementInfo(_ context.Context) []schemaversion.LibraryElementInfo {
+	return []schemaversion.LibraryElementInfo{}
+}
+
 // DataSourceConfig defines different test configurations
 type DataSourceConfig string
 

pkg/registry/apis/dashboard/datasources.go

@@ -4,8 +4,11 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/libraryelements"
+	"github.com/grafana/grafana/pkg/services/libraryelements/model"
 )
 
 type datasourceIndexProvider struct {
@@ -74,3 +77,61 @@ func (d *datasourceIndexProvider) Index(ctx context.Context) *schemaversion.Data
 
 	return index
 }
+
+type libraryElementIndexProvider struct {
+	libraryElementService libraryelements.Service
+}
+
+func (l *libraryElementIndexProvider) GetLibraryElementInfo(ctx context.Context) []schemaversion.LibraryElementInfo {
+	if l.libraryElementService == nil {
+		return []schemaversion.LibraryElementInfo{}
+	}
+
+	nsInfo, err := request.NamespaceInfoFrom(ctx, true)
+	if err != nil {
+		return []schemaversion.LibraryElementInfo{}
+	}
+
+	user := &identity.StaticRequester{
+		OrgID:   nsInfo.OrgID,
+		OrgRole: identity.RoleAdmin,
+	}
+
+	const perPage = 1_000
+	info := make([]schemaversion.LibraryElementInfo, 0)
+	// For some reason the index starts at page 1 here:
+	// https://github.com/grafana/grafana/blob/main/pkg/services/libraryelements/database.go#L418
+	page := 1
+	for {
+		result, err := l.libraryElementService.GetAllElements(ctx, user, model.SearchLibraryElementsQuery{
+			PerPage: perPage,
+			Page:    page,
+		})
+		if err != nil {
+			return []schemaversion.LibraryElementInfo{}
+		}
+
+		for _, elem := range result.Elements {
+			info = append(info, schemaversion.LibraryElementInfo{
+				UID:         elem.UID,
+				Name:        elem.Name,
+				Kind:        elem.Kind,
+				Type:        elem.Type,
+				Description: elem.Description,
+				FolderUID:   elem.FolderUID,
+			})
+		}
+
+		if len(result.Elements) < perPage {
+			break
+		}
+		page++
+
+		// Bound pages to avoid inf loops
+		if page > 100 {
+			break
+		}
+	}
+
+	return info
+}

pkg/registry/apis/dashboard/mutation_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestDashboardAPIBuilder_Mutate(t *testing.T) {
-	migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig))
+	migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig), testutil.NewLibraryElementProvider())
 	tests := []struct {
 		name                string
 		inputObj            runtime.Object

pkg/registry/apis/dashboard/register.go

@@ -177,13 +177,15 @@ func RegisterAPIService(
 	migration.RegisterMetrics(reg)
 	migration.Initialize(&datasourceIndexProvider{
 		datasourceService: datasourceService,
+	}, &libraryElementIndexProvider{
+		libraryElementService: libraryPanels,
 	})
 	apiregistration.RegisterAPI(builder)
 	return builder
 }
 
-func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder {
+func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, libraryElementProvider schemaversion.LibraryElementIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder {
-	migration.Initialize(datasourceProvider)
+	migration.Initialize(datasourceProvider, libraryElementProvider)
 	return &DashboardsAPIBuilder{
 		minRefreshInterval:     "10s",
 		accessClient:           ac,
@@ -231,7 +233,7 @@ func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
 	}
 
 	// Register the explicit conversions
-	if err := conversion.RegisterConversions(scheme, migration.GetDataSourceIndexProvider()); err != nil {
+	if err := conversion.RegisterConversions(scheme, migration.GetDataSourceIndexProvider(), migration.GetLibraryElementIndexProvider()); err != nil {
 		return err
 	}