public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add library elements to dash service (#114016)

Browse Source
This commit is contained in:
Costa Alexoglou
2025-11-18 09:21:05 +01:00
committed by GitHub
parent d6e81c6334
commit faabe2e46d
12 changed files with 162 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/dashboard/pkg/migration/conversion/conversion.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
)
func RegisterConversions(s *runtime.Scheme, dsIndexProvider schemaversion.DataSourceIndexProvider) error {
func RegisterConversions(s *runtime.Scheme, dsIndexProvider schemaversion.DataSourceIndexProvider, _ schemaversion.LibraryElementIndexProvider) error {
// Wrap the provider once with 10s caching for all conversions.
// This prevents repeated DB queries across multiple conversion calls while allowing
// the cache to refresh periodically, making it suitable for long-lived singleton usage.
apps/dashboard/pkg/migration/conversion/conversion_data_loss_detection_test.go
+3 -2
View File
@@ -828,11 +828,12 @@ func TestWithConversionValidation_DataLoss(t *testing.T) {
func TestDataLossDetectionOnAllInputFiles(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := testutil.NewDataSourceProvider(testutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := testutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Set up conversion scheme
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
// Read all files from input directory
apps/dashboard/pkg/migration/conversion/conversion_test.go
+23 -14
View File
@@ -33,7 +33,8 @@ import (
func TestConversionMatrixExist(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
versions := []metav1.Object{
&dashv0.Dashboard{Spec: common.Unstructured{Object: map[string]any{"title": "dashboardV0"}}},
@@ -43,7 +44,7 @@ func TestConversionMatrixExist(t *testing.T) {
}
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
for idx, in := range versions {
@@ -85,11 +86,12 @@ func TestDeepCopyValid(t *testing.T) {
func TestDashboardConversionToAllVersions(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Set up conversion scheme
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
// Read all files from input directory
@@ -244,11 +246,12 @@ func TestDashboardConversionToAllVersions(t *testing.T) {
func TestMigratedDashboardsConversion(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Set up conversion scheme
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
// Read all files from migration package's latest_version directory
@@ -378,7 +381,8 @@ func testConversion(t *testing.T, convertedDash metav1.Object, filename, outputD
func TestConversionMetrics(t *testing.T) {
// Initialize migration with test providers
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Create a test registry for metrics
registry := prometheus.NewRegistry()
@@ -386,7 +390,7 @@ func TestConversionMetrics(t *testing.T) {
// Set up conversion scheme
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
tests := []struct {
@@ -505,7 +509,8 @@ func TestConversionMetrics(t *testing.T) {
// TestConversionMetricsWrapper tests the withConversionMetrics wrapper function
func TestConversionMetricsWrapper(t *testing.T) {
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Create a test registry for metrics
registry := prometheus.NewRegistry()
@@ -673,7 +678,8 @@ func TestSchemaVersionExtraction(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
// Test the schema version extraction logic by creating a wrapper and checking the metrics labels
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Create a test registry for metrics
registry := prometheus.NewRegistry()
@@ -717,7 +723,8 @@ func TestSchemaVersionExtraction(t *testing.T) {
// TestConversionLogging tests that conversion-level logging works correctly
func TestConversionLogging(t *testing.T) {
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
// Create a test registry for metrics
registry := prometheus.NewRegistry()
@@ -725,7 +732,7 @@ func TestConversionLogging(t *testing.T) {
// Set up conversion scheme
scheme := runtime.NewScheme()
err := RegisterConversions(scheme, dsProvider)
err := RegisterConversions(scheme, dsProvider, leProvider)
require.NoError(t, err)
tests := []struct {
@@ -808,7 +815,8 @@ func TestConversionLogging(t *testing.T) {
// TestConversionLogLevels tests that appropriate log levels are used
func TestConversionLogLevels(t *testing.T) {
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("log levels and structured fields verification", func(t *testing.T) {
// Create test wrapper to verify logging behavior
@@ -879,7 +887,8 @@ func TestConversionLogLevels(t *testing.T) {
// TestConversionLoggingFields tests that all expected fields are included in log messages
func TestConversionLoggingFields(t *testing.T) {
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("verify all log fields are present", func(t *testing.T) {
// Test that the conversion wrapper includes all expected structured fields
apps/dashboard/pkg/migration/conversion/v0_test.go
+8 -4
View File
@@ -19,7 +19,8 @@ import (
func TestV0ConversionErrorHandling(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
tests := []struct {
name string
@@ -130,7 +131,8 @@ func TestV0ConversionErrorHandling(t *testing.T) {
func TestV0ConversionErrorPropagation(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("ConvertDashboard_V0_to_V1beta1 returns error on migration failure", func(t *testing.T) {
source := &dashv0.Dashboard{
@@ -203,7 +205,8 @@ func TestV0ConversionErrorPropagation(t *testing.T) {
func TestV0ConversionSuccessPaths(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("Convert_V0_to_V1beta1 success path returns nil", func(t *testing.T) {
source := &dashv0.Dashboard{
@@ -271,7 +274,8 @@ func TestV0ConversionSuccessPaths(t *testing.T) {
func TestV0ConversionSecondStepErrors(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("Convert_V0_to_V2alpha1 sets status on first step error", func(t *testing.T) {
// Create a dashboard that will fail v0->v1beta1 conversion
apps/dashboard/pkg/migration/conversion/v1_test.go
+2 -1
View File
@@ -18,7 +18,8 @@ import (
func TestV1ConversionErrorHandling(t *testing.T) {
// Initialize the migrator with a test data source provider
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
migration.Initialize(dsProvider)
leProvider := migrationtestutil.NewLibraryElementProvider()
migration.Initialize(dsProvider, leProvider)
t.Run("Convert_V1beta1_to_V2alpha1 sets status on conversion error", func(t *testing.T) {
// Create a dashboard that will cause conversion to fail
apps/dashboard/pkg/migration/migrate.go
+14 -4
View File
@@ -9,8 +9,8 @@ import (
)
// Initialize provides the migrator singleton with required dependencies and builds the map of migrations.
func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider) {
migratorInstance.init(dsIndexProvider)
func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
migratorInstance.init(dsIndexProvider, leIndexProvider)
}
// GetDataSourceIndexProvider returns the datasource index provider instance that was initialized.
@@ -20,12 +20,20 @@ func GetDataSourceIndexProvider() schemaversion.DataSourceIndexProvider {
return migratorInstance.dsIndexProvider
}
// GetLibraryElementIndexProvider returns the library element index provider instance that was initialized.
func GetLibraryElementIndexProvider() schemaversion.LibraryElementIndexProvider {
// Wait for initialization to complete
<-migratorInstance.ready
return migratorInstance.leIndexProvider
}
// ResetForTesting resets the migrator singleton for testing purposes.
func ResetForTesting() {
migratorInstance = &migrator{
migrations: map[int]schemaversion.SchemaVersionMigrationFunc{},
ready: make(chan struct{}),
dsIndexProvider: nil,
leIndexProvider: nil,
}
initOnce = sync.Once{}
}
@@ -48,12 +56,14 @@ type migrator struct {
ready chan struct{}
migrations map[int]schemaversion.SchemaVersionMigrationFunc
dsIndexProvider schemaversion.DataSourceIndexProvider
leIndexProvider schemaversion.LibraryElementIndexProvider
}
func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider) {
func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider, leIndexProvider schemaversion.LibraryElementIndexProvider) {
initOnce.Do(func() {
m.dsIndexProvider = dsIndexProvider
m.migrations = schemaversion.GetMigrations(dsIndexProvider)
m.leIndexProvider = leIndexProvider
m.migrations = schemaversion.GetMigrations(dsIndexProvider, leIndexProvider)
close(m.ready)
})
}
apps/dashboard/pkg/migration/migrate_test.go
+15 -5
View File
@@ -29,7 +29,9 @@ const DEV_DASHBOARDS_OUTPUT_DIR = "testdata/dev-dashboards-output"
func TestMigrate(t *testing.T) {
// Reset the migration singleton and use the same datasource provider as the frontend test to ensure consistency
ResetForTesting()
Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
leProvider := migrationtestutil.NewLibraryElementProvider()
Initialize(dsProvider, leProvider)
t.Run("minimum version check", func(t *testing.T) {
err := Migrate(context.Background(), map[string]interface{}{
@@ -45,7 +47,9 @@ func TestMigrate(t *testing.T) {
func TestMigrateSingleVersion(t *testing.T) {
// Use the same datasource provider as the frontend test to ensure consistency
Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
leProvider := migrationtestutil.NewLibraryElementProvider()
Initialize(dsProvider, leProvider)
runSingleVersionMigrationTests(t, SINGLE_VERSION_OUTPUT_DIR)
}
@@ -212,7 +216,9 @@ func loadDashboard(t *testing.T, path string) map[string]interface{} {
// TestSchemaMigrationMetrics tests that schema migration metrics are recorded correctly
func TestSchemaMigrationMetrics(t *testing.T) {
// Initialize migration with test providers
Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
leProvider := migrationtestutil.NewLibraryElementProvider()
Initialize(dsProvider, leProvider)
// Create a test registry for metrics
registry := prometheus.NewRegistry()
@@ -296,7 +302,9 @@ func TestSchemaMigrationMetrics(t *testing.T) {
// TestSchemaMigrationLogging tests that schema migration logging works correctly
func TestSchemaMigrationLogging(t *testing.T) {
Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig))
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.StandardTestConfig)
leProvider := migrationtestutil.NewLibraryElementProvider()
Initialize(dsProvider, leProvider)
tests := []struct {
name string
@@ -413,7 +421,9 @@ func TestMigrateDevDashboards(t *testing.T) {
// Reset the migration singleton and use the dev dashboard datasource provider
// to match the frontend devDashboardDataSources configuration
ResetForTesting()
Initialize(migrationtestutil.NewDataSourceProvider(migrationtestutil.DevDashboardConfig))
dsProvider := migrationtestutil.NewDataSourceProvider(migrationtestutil.DevDashboardConfig)
leProvider := migrationtestutil.NewLibraryElementProvider()
Initialize(dsProvider, leProvider)
runDevDashboardMigrationTests(t, schemaversion.LATEST_VERSION, DEV_DASHBOARDS_OUTPUT_DIR)
}
apps/dashboard/pkg/migration/schemaversion/migrations.go
+16 -1
View File
@@ -27,12 +27,27 @@ type DataSourceIndexProvider interface {
Index(ctx context.Context) *DatasourceIndex
}
type LibraryElementInfo struct {
UID string
Name string
Kind int64
Type string
Description string
FolderUID string
}
type LibraryElementIndexProvider interface {
// GetLibraryElementInfo returns library element information for use in migrations.
GetLibraryElementInfo(ctx context.Context) []LibraryElementInfo
}
type PanelPluginInfo struct {
ID string
Version string
}
func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc {
func GetMigrations(dsIndexProvider DataSourceIndexProvider, _ LibraryElementIndexProvider) map[int]SchemaVersionMigrationFunc {
return map[int]SchemaVersionMigrationFunc{
2: V2,
3: V3,
apps/dashboard/pkg/migration/testutil/mocks.go
+13
View File
@@ -6,6 +6,19 @@ import (
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
)
// EmptyLibraryElementProvider provides an empty library element list for tests
type EmptyLibraryElementProvider struct{}
// NewLibraryElementProvider creates a new empty library element provider for tests
func NewLibraryElementProvider() *EmptyLibraryElementProvider {
return &EmptyLibraryElementProvider{}
}
// GetLibraryElementInfo returns an empty list for tests
func (p *EmptyLibraryElementProvider) GetLibraryElementInfo(_ context.Context) []schemaversion.LibraryElementInfo {
return []schemaversion.LibraryElementInfo{}
}
// DataSourceConfig defines different test configurations
type DataSourceConfig string
pkg/registry/apis/dashboard/datasources.go
+61
View File
@@ -4,8 +4,11 @@ import (
"context"
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/libraryelements"
"github.com/grafana/grafana/pkg/services/libraryelements/model"
)
type datasourceIndexProvider struct {
@@ -74,3 +77,61 @@ func (d *datasourceIndexProvider) Index(ctx context.Context) *schemaversion.Data
return index
}
type libraryElementIndexProvider struct {
libraryElementService libraryelements.Service
}
func (l *libraryElementIndexProvider) GetLibraryElementInfo(ctx context.Context) []schemaversion.LibraryElementInfo {
if l.libraryElementService == nil {
return []schemaversion.LibraryElementInfo{}
}
nsInfo, err := request.NamespaceInfoFrom(ctx, true)
if err != nil {
return []schemaversion.LibraryElementInfo{}
}
user := &identity.StaticRequester{
OrgID: nsInfo.OrgID,
OrgRole: identity.RoleAdmin,
}
const perPage = 1_000
info := make([]schemaversion.LibraryElementInfo, 0)
// For some reason the index starts at page 1 here:
// https://github.com/grafana/grafana/blob/main/pkg/services/libraryelements/database.go#L418
page := 1
for {
result, err := l.libraryElementService.GetAllElements(ctx, user, model.SearchLibraryElementsQuery{
PerPage: perPage,
Page: page,
})
if err != nil {
return []schemaversion.LibraryElementInfo{}
}
for _, elem := range result.Elements {
info = append(info, schemaversion.LibraryElementInfo{
UID: elem.UID,
Name: elem.Name,
Kind: elem.Kind,
Type: elem.Type,
Description: elem.Description,
FolderUID: elem.FolderUID,
})
}
if len(result.Elements) < perPage {
break
}
page++
// Bound pages to avoid inf loops
if page > 100 {
break
}
}
return info
}
pkg/registry/apis/dashboard/mutation_test.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
)
func TestDashboardAPIBuilder_Mutate(t *testing.T) {
migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig))
migration.Initialize(testutil.NewDataSourceProvider(testutil.StandardTestConfig), testutil.NewLibraryElementProvider())
tests := []struct {
name string
inputObj runtime.Object
pkg/registry/apis/dashboard/register.go
+5 -3
View File
@@ -177,13 +177,15 @@ func RegisterAPIService(
migration.RegisterMetrics(reg)
migration.Initialize(&datasourceIndexProvider{
datasourceService: datasourceService,
}, &libraryElementIndexProvider{
libraryElementService: libraryPanels,
})
apiregistration.RegisterAPI(builder)
return builder
}
func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder {
migration.Initialize(datasourceProvider)
func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, libraryElementProvider schemaversion.LibraryElementIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder {
migration.Initialize(datasourceProvider, libraryElementProvider)
return &DashboardsAPIBuilder{
minRefreshInterval: "10s",
accessClient: ac,
@@ -231,7 +233,7 @@ func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
}
// Register the explicit conversions
if err := conversion.RegisterConversions(scheme, migration.GetDataSourceIndexProvider()); err != nil {
if err := conversion.RegisterConversions(scheme, migration.GetDataSourceIndexProvider(), migration.GetLibraryElementIndexProvider()); err != nil {
return err
}