public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dashboards): use index for schema migration datasource lookups (#113911)

Browse Source
This commit is contained in:
Jean-Philippe Quéméner
2025-11-17 08:56:54 +01:00
committed by GitHub
parent f50d513252
commit 64c61c6916
15 changed files with 288 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/dashboard/pkg/migration/schemaversion/README.md
+1 -1
View File
@@ -96,7 +96,7 @@ func V{N}(_ context.Context, dashboard map[string]interface{}) error {
Add your migration to `migrations.go`:
```go
func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionMigrationFunc {
func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc {
return map[int]SchemaVersionMigrationFunc{
// ... existing migrations
{N}: V{N}, // Add your migration here
apps/dashboard/pkg/migration/schemaversion/datasource_utils.go
+105 -28
View File
@@ -1,9 +1,110 @@
package schemaversion
import (
"context"
"sync"
)
// Shared utility functions for datasource migrations across different schema versions.
// These functions handle the common logic for migrating datasource references from
// string names/UIDs to structured reference objects with uid, type, and apiVersion.
// onceIndexProvider wraps a DataSourceIndexProvider to ensure Index() is only called once.
// This prevents multiple DB queries and index builds during operations that may call
// provider.Index() multiple times (e.g., dashboard conversions with many datasource lookups).
//
// Thread-safe: Uses sync.Once to guarantee single execution even under concurrent access.
type onceIndexProvider struct {
provider DataSourceIndexProvider
once sync.Once
index *DatasourceIndex
}
// Index returns the cached index, building it exactly once on first call.
func (p *onceIndexProvider) Index(ctx context.Context) *DatasourceIndex {
p.once.Do(func() {
p.index = p.provider.Index(ctx)
})
return p.index
}
// WrapIndexProviderWithOnce wraps a provider to cache the index for a single operation.
// Useful for conversions or migrations that may call provider.Index() multiple times.
//
// Example usage in dashboard conversion:
//
// onceDsIndexProvider := schemaversion.WrapIndexProviderWithOnce(dsIndexProvider)
// // Now all calls to onceDsIndexProvider.Index(ctx) return the same cached index
func WrapIndexProviderWithOnce(provider DataSourceIndexProvider) DataSourceIndexProvider {
if provider == nil {
return nil
}
return &onceIndexProvider{
provider: provider,
}
}
// DatasourceIndex provides O(1) lookup of datasources by name or UID.
type DatasourceIndex struct {
ByName map[string]*DataSourceInfo
ByUID map[string]*DataSourceInfo
DefaultDS *DataSourceInfo
}
// NewDatasourceIndex creates an index from a list of datasources.
// Iterates once through the list to build name and UID maps for O(1) lookups.
func NewDatasourceIndex(datasources []DataSourceInfo) *DatasourceIndex {
idx := &DatasourceIndex{
ByName: make(map[string]*DataSourceInfo, len(datasources)),
ByUID: make(map[string]*DataSourceInfo, len(datasources)),
}
for i := range datasources {
ds := &datasources[i]
// Index by name if present
if ds.Name != "" {
idx.ByName[ds.Name] = ds
}
// Index by UID if present
if ds.UID != "" {
idx.ByUID[ds.UID] = ds
}
// Track default datasource
if ds.Default {
idx.DefaultDS = ds
}
}
return idx
}
// Lookup finds a datasource by name or UID string.
// Returns the datasource info if found, nil otherwise.
func (idx *DatasourceIndex) Lookup(nameOrUID string) *DataSourceInfo {
// Try name first (most common in legacy dashboards)
if ds := idx.ByName[nameOrUID]; ds != nil {
return ds
}
// Try UID second
return idx.ByUID[nameOrUID]
}
func (idx *DatasourceIndex) LookupByUID(uid string) *DataSourceInfo {
return idx.ByUID[uid]
}
func (idx *DatasourceIndex) LookupByName(name string) *DataSourceInfo {
return idx.ByName[name]
}
// GetDefault returns the default datasource, if one exists.
func (idx *DatasourceIndex) GetDefault() *DataSourceInfo {
return idx.DefaultDS
}
// GetDataSourceRef creates a datasource reference object with uid, type and optional apiVersion
func GetDataSourceRef(ds *DataSourceInfo) map[string]interface{} {
if ds == nil {
@@ -19,21 +120,6 @@ func GetDataSourceRef(ds *DataSourceInfo) map[string]interface{} {
return ref
}
// GetDefaultDSInstanceSettings returns the default datasource if one exists
func GetDefaultDSInstanceSettings(datasources []DataSourceInfo) *DataSourceInfo {
for _, ds := range datasources {
if ds.Default {
return &DataSourceInfo{
UID: ds.UID,
Type: ds.Type,
Name: ds.Name,
APIVersion: ds.APIVersion,
}
}
}
return nil
}
// isDataSourceRef checks if the object is a valid DataSourceRef (has uid or type)
// Matches the frontend isDataSourceRef function in datasource.ts
func isDataSourceRef(ref interface{}) bool {
@@ -64,7 +150,7 @@ func isDataSourceRef(ref interface{}) bool {
// Options:
// - returnDefaultAsNull: if true, returns nil for "default" datasources (used in V33)
// - returnDefaultAsNull: if false, returns reference for "default" datasources (used in V36)
func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, datasources []DataSourceInfo) map[string]interface{} {
func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, index *DatasourceIndex) map[string]interface{} {
if options["returnDefaultAsNull"] && (nameOrRef == nil || nameOrRef == "default") {
return nil
}
@@ -76,8 +162,7 @@ func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool,
// Look up datasource by name/UID
if nameOrRef == nil || nameOrRef == "default" {
ds := GetDefaultDSInstanceSettings(datasources)
if ds != nil {
if ds := index.GetDefault(); ds != nil {
return GetDataSourceRef(ds)
}
}
@@ -90,16 +175,8 @@ func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool,
return map[string]interface{}{}
}
// Search for matching datasource
for _, ds := range datasources {
if str == ds.Name || str == ds.UID {
return GetDataSourceRef(&DataSourceInfo{
UID: ds.UID,
Type: ds.Type,
Name: ds.Name,
APIVersion: ds.APIVersion,
})
}
if ds := index.Lookup(str); ds != nil {
return GetDataSourceRef(ds)
}
// Unknown datasource name should be preserved as UID-only reference
apps/dashboard/pkg/migration/schemaversion/datasource_utils_test.go
+7 -61
View File
@@ -67,68 +67,13 @@ func TestGetDataSourceRef(t *testing.T) {
}
}
func TestGetDefaultDSInstanceSettings(t *testing.T) {
tests := []struct {
name string
datasources []schemaversion.DataSourceInfo
expected *schemaversion.DataSourceInfo
}{
{
name: "empty datasources list",
datasources: []schemaversion.DataSourceInfo{},
expected: nil,
},
{
name: "no default datasource",
datasources: []schemaversion.DataSourceInfo{
{UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false},
{UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false},
},
expected: nil,
},
{
name: "single default datasource",
datasources: []schemaversion.DataSourceInfo{
{UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false},
{UID: "default-ds-uid", Type: "prometheus", Name: "Default Test Datasource Name", Default: true, APIVersion: "v1"},
{UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false},
},
expected: &schemaversion.DataSourceInfo{
UID: "default-ds-uid",
Type: "prometheus",
Name: "Default Test Datasource Name",
APIVersion: "v1",
},
},
{
name: "multiple default datasources returns first",
datasources: []schemaversion.DataSourceInfo{
{UID: "first-default", Type: "prometheus", Name: "First Default", Default: true, APIVersion: "v1"},
{UID: "second-default", Type: "elasticsearch", Name: "Second Default", Default: true, APIVersion: "v2"},
},
expected: &schemaversion.DataSourceInfo{
UID: "first-default",
Type: "prometheus",
Name: "First Default",
APIVersion: "v1",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := schemaversion.GetDefaultDSInstanceSettings(tt.datasources)
assert.Equal(t, tt.expected, result)
})
}
}
func TestMigrateDatasourceNameToRef(t *testing.T) {
datasources := []schemaversion.DataSourceInfo{
{UID: "default-ds-uid", Type: "prometheus", Name: "Default Test Datasource Name", Default: true, APIVersion: "v1"},
{UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false, APIVersion: "v2"},
{UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false, APIVersion: "v1"},
}
index := schemaversion.NewDatasourceIndex(datasources)
t.Run("returnDefaultAsNull: true", func(t *testing.T) {
options := map[string]bool{"returnDefaultAsNull": true}
@@ -193,7 +138,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, datasources)
result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, index)
assert.Equal(t, tt.expected, result)
})
}
@@ -261,7 +206,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, datasources)
result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, index)
assert.Equal(t, tt.expected, result)
})
}
@@ -274,7 +219,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) {
nameOrRef := map[string]interface{}{
"type": "prometheus",
}
result := schemaversion.MigrateDatasourceNameToRef(nameOrRef, options, datasources)
result := schemaversion.MigrateDatasourceNameToRef(nameOrRef, options, index)
expected := map[string]interface{}{
"type": "prometheus",
}
@@ -282,13 +227,14 @@ func TestMigrateDatasourceNameToRef(t *testing.T) {
})
t.Run("integer input should return nil", func(t *testing.T) {
result := schemaversion.MigrateDatasourceNameToRef(123, options, datasources)
result := schemaversion.MigrateDatasourceNameToRef(123, options, index)
expected := map[string]interface{}(nil)
assert.Equal(t, expected, result)
})
t.Run("empty datasources list", func(t *testing.T) {
result := schemaversion.MigrateDatasourceNameToRef("any-ds", options, []schemaversion.DataSourceInfo{})
emptyIndex := schemaversion.NewDatasourceIndex([]schemaversion.DataSourceInfo{})
result := schemaversion.MigrateDatasourceNameToRef("any-ds", options, emptyIndex)
expected := map[string]interface{}{
"uid": "any-ds",
}
apps/dashboard/pkg/migration/schemaversion/migrations.go
+7 -7
View File
@@ -22,10 +22,10 @@ type DataSourceInfo struct {
APIVersion string
}
type DataSourceInfoProvider interface {
// GetDataSourceInfo returns a list of all data sources with their info
// The context must have the namespace in it
GetDataSourceInfo(ctx context.Context) []DataSourceInfo
type DataSourceIndexProvider interface {
// Index returns a pre-built index for O(1) datasource lookups.
Index(ctx context.Context) *DatasourceIndex
}
type PanelPluginInfo struct {
@@ -33,7 +33,7 @@ type PanelPluginInfo struct {
Version string
}
func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionMigrationFunc {
func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc {
return map[int]SchemaVersionMigrationFunc{
2: V2,
3: V3,
@@ -66,10 +66,10 @@ func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionM
30: V30,
31: V31,
32: V32,
33: V33(dsInfoProvider),
33: V33(dsIndexProvider),
34: V34,
35: V35,
36: V36(dsInfoProvider),
36: V36(dsIndexProvider),
37: V37,
38: V38,
39: V39,
apps/dashboard/pkg/migration/schemaversion/v33.go
+11 -9
View File
@@ -60,22 +60,24 @@ import (
// { "refId": "A", "datasource": "default" }
// ]
// }
func V33(dsInfo DataSourceInfoProvider) SchemaVersionMigrationFunc {
func V33(dsIndexProvider DataSourceIndexProvider) SchemaVersionMigrationFunc {
return func(ctx context.Context, dashboard map[string]interface{}) error {
datasources := dsInfo.GetDataSourceInfo(ctx)
if dashboard == nil {
dashboard = map[string]interface{}{}
}
dashboard["schemaVersion"] = int(33)
migratePanelsV33(dashboard, datasources)
// Build datasource index directly from provider for O(1) lookups
index := dsIndexProvider.Index(ctx)
migratePanelsV33(dashboard, index)
return nil
}
}
// migratePanelsV33 updates datasource references in dashboard panels for V33 migration
func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSourceInfo) {
func migratePanelsV33(dashboard map[string]interface{}, index *DatasourceIndex) {
if dashboard == nil {
return
}
@@ -90,7 +92,7 @@ func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSource
continue
}
migratePanelDatasourcesV33(panelMap, datasources)
migratePanelDatasourcesV33(panelMap, index)
// Handle nested panels in collapsed rows
nestedPanels, hasNested := panelMap["panels"].([]interface{})
@@ -103,15 +105,15 @@ func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSource
if !ok {
continue
}
migratePanelDatasourcesV33(np, datasources)
migratePanelDatasourcesV33(np, index)
}
}
}
// migratePanelDatasourcesV33 updates datasource references in a single panel and its targets for V33 migration
func migratePanelDatasourcesV33(panelMap map[string]interface{}, datasources []DataSourceInfo) {
func migratePanelDatasourcesV33(panelMap map[string]interface{}, index *DatasourceIndex) {
// Handle panel datasource - always set result (even if nil)
if result := MigrateDatasourceNameToRef(panelMap["datasource"], map[string]bool{"returnDefaultAsNull": true}, datasources); result != nil {
if result := MigrateDatasourceNameToRef(panelMap["datasource"], map[string]bool{"returnDefaultAsNull": true}, index); result != nil {
panelMap["datasource"] = result
} else {
panelMap["datasource"] = nil
@@ -135,7 +137,7 @@ func migratePanelDatasourcesV33(panelMap map[string]interface{}, datasources []D
}
// Only set target datasource if migration result is not nil
if targetRef := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": true}, datasources); targetRef != nil {
if targetRef := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": true}, index); targetRef != nil {
targetMap["datasource"] = targetRef
}
// If targetRef is nil, leave target.datasource unchanged (preserves "default" strings, etc.)
apps/dashboard/pkg/migration/schemaversion/v33_test.go
+4
View File
@@ -222,3 +222,7 @@ type testDataSourceProvider struct {
func (p *testDataSourceProvider) GetDataSourceInfo(_ context.Context) []DataSourceInfo {
return p.datasources
}
func (p *testDataSourceProvider) Index(_ context.Context) *DatasourceIndex {
return NewDatasourceIndex(p.datasources)
}
apps/dashboard/pkg/migration/schemaversion/v36.go
+19 -19
View File
@@ -76,21 +76,21 @@ import (
// refId: "A"
// }]
// }
func V36(dsInfo DataSourceInfoProvider) SchemaVersionMigrationFunc {
func V36(dsIndexProvider DataSourceIndexProvider) SchemaVersionMigrationFunc {
return func(ctx context.Context, dashboard map[string]interface{}) error {
datasources := dsInfo.GetDataSourceInfo(ctx)
dsIndex := dsIndexProvider.Index(ctx)
dashboard["schemaVersion"] = int(36)
migrateAnnotations(dashboard, datasources)
migrateTemplateVariables(dashboard, datasources)
migratePanels(dashboard, datasources)
migrateAnnotations(dashboard, dsIndex)
migrateTemplateVariables(dashboard, dsIndex)
migratePanels(dashboard, dsIndex)
return nil
}
}
// migrateAnnotations updates datasource references in dashboard annotations
func migrateAnnotations(dashboard map[string]interface{}, datasources []DataSourceInfo) {
func migrateAnnotations(dashboard map[string]interface{}, index *DatasourceIndex) {
annotations, ok := dashboard["annotations"].(map[string]interface{})
if !ok {
return
@@ -109,12 +109,12 @@ func migrateAnnotations(dashboard map[string]interface{}, datasources []DataSour
// Always migrate datasource, even if it doesn't exist (will be set to default)
ds := queryMap["datasource"]
queryMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources)
queryMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index)
}
}
// migrateTemplateVariables updates datasource references in dashboard variables
func migrateTemplateVariables(dashboard map[string]interface{}, datasources []DataSourceInfo) {
func migrateTemplateVariables(dashboard map[string]interface{}, index *DatasourceIndex) {
templating, ok := dashboard["templating"].(map[string]interface{})
if !ok {
return
@@ -125,7 +125,7 @@ func migrateTemplateVariables(dashboard map[string]interface{}, datasources []Da
return
}
defaultDS := GetDefaultDSInstanceSettings(datasources)
defaultDS := index.GetDefault()
for _, variable := range list {
varMap, ok := variable.(map[string]interface{})
if !ok {
@@ -149,7 +149,7 @@ func migrateTemplateVariables(dashboard map[string]interface{}, datasources []Da
}
// migratePanels updates datasource references in dashboard panels
func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInfo) {
func migratePanels(dashboard map[string]interface{}, index *DatasourceIndex) {
panels, ok := dashboard["panels"].([]interface{})
if !ok {
return
@@ -160,7 +160,7 @@ func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInf
if !ok {
continue
}
migratePanelDatasources(panelMap, datasources)
migratePanelDatasources(panelMap, index)
// Handle nested panels in collapsed rows
nestedPanels, hasNested := panelMap["panels"].([]interface{})
@@ -173,24 +173,24 @@ func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInf
if !ok {
continue
}
migratePanelDatasourcesInternal(np, datasources, true)
migratePanelDatasourcesInternal(np, index, true)
}
}
}
// migratePanelDatasources updates datasource references in a single panel and its targets
func migratePanelDatasources(panelMap map[string]interface{}, datasources []DataSourceInfo) {
migratePanelDatasourcesInternal(panelMap, datasources, false)
func migratePanelDatasources(panelMap map[string]interface{}, index *DatasourceIndex) {
migratePanelDatasourcesInternal(panelMap, index, false)
}
// migratePanelDatasourcesInternal updates datasource references with nesting awareness
func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasources []DataSourceInfo, isNested bool) {
func migratePanelDatasourcesInternal(panelMap map[string]interface{}, index *DatasourceIndex, isNested bool) {
// NOTE: Even though row panels don't technically need datasource or targets fields,
// we process them anyway to exactly match frontend behavior and avoid inconsistencies
// between frontend and backend migrations. The frontend DashboardMigrator processes
// all panels uniformly without special row panel handling.
defaultDS := GetDefaultDSInstanceSettings(datasources)
defaultDS := index.GetDefault()
panelDataSourceWasDefault := false
// Handle targets - only add default targets to top-level panels (matches frontend behavior)
@@ -228,7 +228,7 @@ func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasource
// Keep empty object {} as-is (set by V33 migration for empty strings)
panelMap["datasource"] = ds
} else {
migrated := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources)
migrated := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index)
panelMap["datasource"] = migrated
}
}
@@ -273,12 +273,12 @@ func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasource
targetMap["datasource"] = result
} else {
// Frontend: target.datasource = migrateDatasourceNameToRef(target.datasource, { returnDefaultAsNull: false });
targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources)
targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index)
}
}
} else {
// Migrate existing target datasource
targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources)
targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index)
}
// Update panel datasource if it was default and target is not an expression