From 64c61c691632d5406d2654fd65bb58afc4374c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Philippe=20Qu=C3=A9m=C3=A9ner?= Date: Mon, 17 Nov 2025 08:56:54 +0100 Subject: [PATCH] fix(dashboards): use index for schema migration datasource lookups (#113911) --- .../pkg/migration/conversion/conversion.go | 10 +- apps/dashboard/pkg/migration/conversion/v0.go | 8 +- apps/dashboard/pkg/migration/conversion/v1.go | 8 +- .../conversion/v1beta1_to_v2alpha1.go | 117 +++++++-------- apps/dashboard/pkg/migration/migrate.go | 29 ++-- .../pkg/migration/schemaversion/README.md | 2 +- .../schemaversion/datasource_utils.go | 133 ++++++++++++++---- .../schemaversion/datasource_utils_test.go | 68 +-------- .../pkg/migration/schemaversion/migrations.go | 14 +- .../pkg/migration/schemaversion/v33.go | 20 +-- .../pkg/migration/schemaversion/v33_test.go | 4 + .../pkg/migration/schemaversion/v36.go | 38 ++--- .../dashboard/pkg/migration/testutil/mocks.go | 6 + pkg/registry/apis/dashboard/datasources.go | 50 +++++-- pkg/registry/apis/dashboard/register.go | 6 +- 15 files changed, 288 insertions(+), 225 deletions(-) diff --git a/apps/dashboard/pkg/migration/conversion/conversion.go b/apps/dashboard/pkg/migration/conversion/conversion.go index 90795b7779a..9ee376a15ba 100644 --- a/apps/dashboard/pkg/migration/conversion/conversion.go +++ b/apps/dashboard/pkg/migration/conversion/conversion.go @@ -11,7 +11,7 @@ import ( "github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion" ) -func RegisterConversions(s *runtime.Scheme, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func RegisterConversions(s *runtime.Scheme, dsIndexProvider schemaversion.DataSourceIndexProvider) error { // v0 conversions if err := s.AddConversionFunc((*dashv0.Dashboard)(nil), (*dashv1.Dashboard)(nil), withConversionMetrics(dashv0.APIVERSION, dashv1.APIVERSION, func(a, b interface{}, scope conversion.Scope) error { @@ -21,13 +21,13 @@ func RegisterConversions(s *runtime.Scheme, dsInfoProvider schemaversion.DataSou } if err := s.AddConversionFunc((*dashv0.Dashboard)(nil), (*dashv2alpha1.Dashboard)(nil), withConversionMetrics(dashv0.APIVERSION, dashv2alpha1.APIVERSION, func(a, b interface{}, scope conversion.Scope) error { - return Convert_V0_to_V2alpha1(a.(*dashv0.Dashboard), b.(*dashv2alpha1.Dashboard), scope, dsInfoProvider) + return Convert_V0_to_V2alpha1(a.(*dashv0.Dashboard), b.(*dashv2alpha1.Dashboard), scope, dsIndexProvider) })); err != nil { return err } if err := s.AddConversionFunc((*dashv0.Dashboard)(nil), (*dashv2beta1.Dashboard)(nil), withConversionMetrics(dashv0.APIVERSION, dashv2beta1.APIVERSION, func(a, b interface{}, scope conversion.Scope) error { - return Convert_V0_to_V2beta1(a.(*dashv0.Dashboard), b.(*dashv2beta1.Dashboard), scope, dsInfoProvider) + return Convert_V0_to_V2beta1(a.(*dashv0.Dashboard), b.(*dashv2beta1.Dashboard), scope, dsIndexProvider) })); err != nil { return err } @@ -41,13 +41,13 @@ func RegisterConversions(s *runtime.Scheme, dsInfoProvider schemaversion.DataSou } if err := s.AddConversionFunc((*dashv1.Dashboard)(nil), (*dashv2alpha1.Dashboard)(nil), withConversionMetrics(dashv1.APIVERSION, dashv2alpha1.APIVERSION, func(a, b interface{}, scope conversion.Scope) error { - return Convert_V1beta1_to_V2alpha1(a.(*dashv1.Dashboard), b.(*dashv2alpha1.Dashboard), scope, dsInfoProvider) + return Convert_V1beta1_to_V2alpha1(a.(*dashv1.Dashboard), b.(*dashv2alpha1.Dashboard), scope, dsIndexProvider) })); err != nil { return err } if err := s.AddConversionFunc((*dashv1.Dashboard)(nil), (*dashv2beta1.Dashboard)(nil), withConversionMetrics(dashv1.APIVERSION, dashv2beta1.APIVERSION, func(a, b interface{}, scope conversion.Scope) error { - return Convert_V1beta1_to_V2beta1(a.(*dashv1.Dashboard), b.(*dashv2beta1.Dashboard), scope, dsInfoProvider) + return Convert_V1beta1_to_V2beta1(a.(*dashv1.Dashboard), b.(*dashv2beta1.Dashboard), scope, dsIndexProvider) })); err != nil { return err } diff --git a/apps/dashboard/pkg/migration/conversion/v0.go b/apps/dashboard/pkg/migration/conversion/v0.go index 81ea2db5e60..e182a71a52a 100644 --- a/apps/dashboard/pkg/migration/conversion/v0.go +++ b/apps/dashboard/pkg/migration/conversion/v0.go @@ -25,7 +25,7 @@ func Convert_V0_to_V1beta1(in *dashv0.Dashboard, out *dashv1.Dashboard, scope co return nil } -func Convert_V0_to_V2alpha1(in *dashv0.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func Convert_V0_to_V2alpha1(in *dashv0.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsIndexProvider schemaversion.DataSourceIndexProvider) error { v1beta1 := &dashv1.Dashboard{} if err := ConvertDashboard_V0_to_V1beta1(in, v1beta1, scope); err != nil { out.Status = dashv2alpha1.DashboardStatus{ @@ -48,7 +48,7 @@ func Convert_V0_to_V2alpha1(in *dashv0.Dashboard, out *dashv2alpha1.Dashboard, s return nil } - if err := ConvertDashboard_V1beta1_to_V2alpha1(v1beta1, out, scope, dsInfoProvider); err != nil { + if err := ConvertDashboard_V1beta1_to_V2alpha1(v1beta1, out, scope, dsIndexProvider); err != nil { out.Status = dashv2alpha1.DashboardStatus{ Conversion: &dashv2alpha1.DashboardConversionStatus{ StoredVersion: ptr.To(dashv0.VERSION), @@ -72,7 +72,7 @@ func Convert_V0_to_V2alpha1(in *dashv0.Dashboard, out *dashv2alpha1.Dashboard, s return nil } -func Convert_V0_to_V2beta1(in *dashv0.Dashboard, out *dashv2beta1.Dashboard, scope conversion.Scope, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func Convert_V0_to_V2beta1(in *dashv0.Dashboard, out *dashv2beta1.Dashboard, scope conversion.Scope, dsIndexProvider schemaversion.DataSourceIndexProvider) error { v1beta1 := &dashv1.Dashboard{} if err := ConvertDashboard_V0_to_V1beta1(in, v1beta1, scope); err != nil { out.Status = dashv2beta1.DashboardStatus{ @@ -86,7 +86,7 @@ func Convert_V0_to_V2beta1(in *dashv0.Dashboard, out *dashv2beta1.Dashboard, sco } v2alpha1 := &dashv2alpha1.Dashboard{} - if err := ConvertDashboard_V1beta1_to_V2alpha1(v1beta1, v2alpha1, scope, dsInfoProvider); err != nil { + if err := ConvertDashboard_V1beta1_to_V2alpha1(v1beta1, v2alpha1, scope, dsIndexProvider); err != nil { out.Status = dashv2beta1.DashboardStatus{ Conversion: &dashv2beta1.DashboardConversionStatus{ StoredVersion: ptr.To(dashv0.VERSION), diff --git a/apps/dashboard/pkg/migration/conversion/v1.go b/apps/dashboard/pkg/migration/conversion/v1.go index 107d3010a9a..48fdbf93b36 100644 --- a/apps/dashboard/pkg/migration/conversion/v1.go +++ b/apps/dashboard/pkg/migration/conversion/v1.go @@ -25,8 +25,8 @@ func Convert_V1beta1_to_V0(in *dashv1.Dashboard, out *dashv0.Dashboard, scope co return nil } -func Convert_V1beta1_to_V2alpha1(in *dashv1.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsInfoProvider schemaversion.DataSourceInfoProvider) error { - if err := ConvertDashboard_V1beta1_to_V2alpha1(in, out, scope, dsInfoProvider); err != nil { +func Convert_V1beta1_to_V2alpha1(in *dashv1.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsIndexProvider schemaversion.DataSourceIndexProvider) error { + if err := ConvertDashboard_V1beta1_to_V2alpha1(in, out, scope, dsIndexProvider); err != nil { out.Status = dashv2alpha1.DashboardStatus{ Conversion: &dashv2alpha1.DashboardConversionStatus{ StoredVersion: ptr.To(dashv1.VERSION), @@ -60,9 +60,9 @@ func Convert_V1beta1_to_V2alpha1(in *dashv1.Dashboard, out *dashv2alpha1.Dashboa return nil } -func Convert_V1beta1_to_V2beta1(in *dashv1.Dashboard, out *dashv2beta1.Dashboard, scope conversion.Scope, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func Convert_V1beta1_to_V2beta1(in *dashv1.Dashboard, out *dashv2beta1.Dashboard, scope conversion.Scope, dsIndexProvider schemaversion.DataSourceIndexProvider) error { v2alpha1 := &dashv2alpha1.Dashboard{} - if err := ConvertDashboard_V1beta1_to_V2alpha1(in, v2alpha1, scope, dsInfoProvider); err != nil { + if err := ConvertDashboard_V1beta1_to_V2alpha1(in, v2alpha1, scope, dsIndexProvider); err != nil { out.Status = dashv2beta1.DashboardStatus{ Conversion: &dashv2beta1.DashboardConversionStatus{ StoredVersion: ptr.To(dashv1.VERSION), diff --git a/apps/dashboard/pkg/migration/conversion/v1beta1_to_v2alpha1.go b/apps/dashboard/pkg/migration/conversion/v1beta1_to_v2alpha1.go index c514728509e..76a630ac0b6 100644 --- a/apps/dashboard/pkg/migration/conversion/v1beta1_to_v2alpha1.go +++ b/apps/dashboard/pkg/migration/conversion/v1beta1_to_v2alpha1.go @@ -19,25 +19,23 @@ import ( ) // getDefaultDatasourceType gets the default datasource type using the datasource provider -func getDefaultDatasourceType(ctx context.Context, provider schemaversion.DataSourceInfoProvider) string { +func getDefaultDatasourceType(ctx context.Context, provider schemaversion.DataSourceIndexProvider) string { const defaultType = "grafana" if provider == nil { return defaultType } - datasources := provider.GetDataSourceInfo(ctx) - for _, ds := range datasources { - if ds.Default { - return ds.Type - } + datasources := provider.Index(ctx) + if defaultDS := datasources.GetDefault(); defaultDS != nil { + return defaultDS.Type } return defaultType } // getDatasourceTypeByUID gets the datasource type by UID using the datasource provider -func getDatasourceTypeByUID(ctx context.Context, uid string, provider schemaversion.DataSourceInfoProvider) string { +func getDatasourceTypeByUID(ctx context.Context, uid string, provider schemaversion.DataSourceIndexProvider) string { if uid == "" { return getDefaultDatasourceType(ctx, provider) } @@ -46,11 +44,9 @@ func getDatasourceTypeByUID(ctx context.Context, uid string, provider schemavers return "grafana" } - datasources := provider.GetDataSourceInfo(ctx) - for _, ds := range datasources { - if ds.UID == uid { - return ds.Type - } + dsIndex := provider.Index(ctx) + if ds := dsIndex.LookupByUID(uid); ds != nil { + return ds.Type } return getDefaultDatasourceType(ctx, provider) @@ -62,9 +58,9 @@ func getDatasourceTypeByUID(ctx context.Context, uid string, provider schemavers // A background service identity is used because the user who is reading the specific dashboard // may not have access to all the datasources in the dashboard, but the conversion still needs to take place // in order to be able to convert between k8s versions. -func prepareV1beta1ConversionContext(in *dashv1.Dashboard, dsInfoProvider schemaversion.DataSourceInfoProvider) (context.Context, *types.NamespaceInfo, error) { - if dsInfoProvider == nil { - return nil, nil, fmt.Errorf("datasource provider not initialized") +func prepareV1beta1ConversionContext(in *dashv1.Dashboard, dsIndexProvider schemaversion.DataSourceIndexProvider) (context.Context, *types.NamespaceInfo, error) { + if dsIndexProvider == nil { + return nil, nil, fmt.Errorf("datasource index provider not initialized") } namespace := in.GetNamespace() @@ -84,24 +80,29 @@ func prepareV1beta1ConversionContext(in *dashv1.Dashboard, dsInfoProvider schema return ctx, &nsInfo, nil } -func ConvertDashboard_V1beta1_to_V2alpha1(in *dashv1.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func ConvertDashboard_V1beta1_to_V2alpha1(in *dashv1.Dashboard, out *dashv2alpha1.Dashboard, scope conversion.Scope, dsIndexProvider schemaversion.DataSourceIndexProvider) error { out.ObjectMeta = in.ObjectMeta out.APIVersion = dashv2alpha1.APIVERSION out.Kind = in.Kind + // Wrap the provider to ensure Index() is called only once during this conversion. + // This prevents multiple DB queries and index builds when the provider is used in multiple places + // (e.g., getDefaultDatasourceType, getDatasourceTypeByUID, panel datasource conversions, etc.) + dsIndexProviderWrapped := schemaversion.WrapIndexProviderWithOnce(dsIndexProvider) + // Prepare context with namespace and service identity // The datasource provider is passed as a parameter (captured in closure when conversions are registered) - ctx, _, err := prepareV1beta1ConversionContext(in, dsInfoProvider) + ctx, _, err := prepareV1beta1ConversionContext(in, dsIndexProviderWrapped) if err != nil { // If context preparation fails, return error to be handled by wrapper // The wrapper will set status and handle gracefully return fmt.Errorf("failed to prepare conversion context: %w", err) } - return convertDashboardSpec_V1beta1_to_V2alpha1(&in.Spec, &out.Spec, scope, ctx, dsInfoProvider) + return convertDashboardSpec_V1beta1_to_V2alpha1(&in.Spec, &out.Spec, scope, ctx, dsIndexProviderWrapped) } -func convertDashboardSpec_V1beta1_to_V2alpha1(in *dashv1.DashboardSpec, out *dashv2alpha1.DashboardSpec, scope conversion.Scope, ctx context.Context, dsInfoProvider schemaversion.DataSourceInfoProvider) error { +func convertDashboardSpec_V1beta1_to_V2alpha1(in *dashv1.DashboardSpec, out *dashv2alpha1.DashboardSpec, scope conversion.Scope, ctx context.Context, dsIndexProvider schemaversion.DataSourceIndexProvider) error { // Parse the unstructured spec into a dashboard JSON structure dashboardJSON, ok := in.Object["dashboard"] if !ok { @@ -165,7 +166,7 @@ func convertDashboardSpec_V1beta1_to_V2alpha1(in *dashv1.DashboardSpec, out *das out.Links = transformLinks(dashboard) // Transform panels to elements and layout - elements, layout, err := transformPanelsToElementsAndLayout(ctx, dashboard, dsInfoProvider) + elements, layout, err := transformPanelsToElementsAndLayout(ctx, dashboard, dsIndexProvider) if err != nil { return fmt.Errorf("failed to transform panels: %w", err) } @@ -173,7 +174,7 @@ func convertDashboardSpec_V1beta1_to_V2alpha1(in *dashv1.DashboardSpec, out *das out.Layout = layout // Transform variables - variables, err := transformVariables(ctx, dashboard, dsInfoProvider) + variables, err := transformVariables(ctx, dashboard, dsIndexProvider) if err != nil { return fmt.Errorf("failed to transform variables: %w", err) } @@ -391,7 +392,7 @@ func transformLinks(dashboard map[string]interface{}) []dashv2alpha1.DashboardDa // Panel transformation constants const GRID_ROW_HEIGHT = 1 -func transformPanelsToElementsAndLayout(ctx context.Context, dashboard map[string]interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { +func transformPanelsToElementsAndLayout(ctx context.Context, dashboard map[string]interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { panels, ok := dashboard["panels"].([]interface{}) if !ok { // Return empty elements and default grid layout @@ -419,13 +420,13 @@ func transformPanelsToElementsAndLayout(ctx context.Context, dashboard map[strin } if hasRowPanels { - return convertToRowsLayout(ctx, panels, dsInfoProvider) + return convertToRowsLayout(ctx, panels, dsIndexProvider) } - return convertToGridLayout(ctx, panels, dsInfoProvider) + return convertToGridLayout(ctx, panels, dsIndexProvider) } -func convertToGridLayout(ctx context.Context, panels []interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { +func convertToGridLayout(ctx context.Context, panels []interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { elements := make(map[string]dashv2alpha1.DashboardElement) items := make([]dashv2alpha1.DashboardGridLayoutItemKind, 0, len(panels)) @@ -435,7 +436,7 @@ func convertToGridLayout(ctx context.Context, panels []interface{}, dsInfoProvid continue } - element, elementName, err := buildElement(ctx, panelMap, dsInfoProvider) + element, elementName, err := buildElement(ctx, panelMap, dsIndexProvider) if err != nil { continue // Skip invalid panels } @@ -456,7 +457,7 @@ func convertToGridLayout(ctx context.Context, panels []interface{}, dsInfoProvid return elements, layout, nil } -func convertToRowsLayout(ctx context.Context, panels []interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { +func convertToRowsLayout(ctx context.Context, panels []interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) (map[string]dashv2alpha1.DashboardElement, dashv2alpha1.DashboardGridLayoutKindOrRowsLayoutKindOrAutoGridLayoutKindOrTabsLayoutKind, error) { elements := make(map[string]dashv2alpha1.DashboardElement) rows := make([]dashv2alpha1.DashboardRowsLayoutRowKind, 0) @@ -492,7 +493,7 @@ func convertToRowsLayout(ctx context.Context, panels []interface{}, dsInfoProvid if collapsedPanels, ok := panelMap["panels"].([]interface{}); ok { for _, panel := range collapsedPanels { if collapsedPanelMap, ok := panel.(map[string]interface{}); ok { - element, name, err := buildElement(ctx, collapsedPanelMap, dsInfoProvider) + element, name, err := buildElement(ctx, collapsedPanelMap, dsIndexProvider) if err == nil { elements[name] = element rowElements = append(rowElements, buildGridItemKind(collapsedPanelMap, name, int64Ptr(yOffsetInRows(collapsedPanelMap, legacyRowY)))) @@ -504,7 +505,7 @@ func convertToRowsLayout(ctx context.Context, panels []interface{}, dsInfoProvid currentRow = buildRowKind(panelMap, rowElements) } else { // Regular panel - element, elementName, err := buildElement(ctx, panelMap, dsInfoProvider) + element, elementName, err := buildElement(ctx, panelMap, dsIndexProvider) if err != nil { continue // Skip invalid panels } @@ -566,7 +567,7 @@ func convertToRowsLayout(ctx context.Context, panels []interface{}, dsInfoProvid return elements, layout, nil } -func buildElement(ctx context.Context, panelMap map[string]interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) (dashv2alpha1.DashboardElement, string, error) { +func buildElement(ctx context.Context, panelMap map[string]interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) (dashv2alpha1.DashboardElement, string, error) { panelID := getIntField(panelMap, "id", 0) elementName := fmt.Sprintf("panel-%d", panelID) @@ -592,7 +593,7 @@ func buildElement(ctx context.Context, panelMap map[string]interface{}, dsInfoPr } // Regular panel - panelKind, err := buildPanelKind(ctx, panelMap, dsInfoProvider) + panelKind, err := buildPanelKind(ctx, panelMap, dsIndexProvider) if err != nil { return dashv2alpha1.DashboardElement{}, "", err } @@ -604,11 +605,11 @@ func buildElement(ctx context.Context, panelMap map[string]interface{}, dsInfoPr return element, elementName, nil } -func buildPanelKind(ctx context.Context, panelMap map[string]interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) (*dashv2alpha1.DashboardPanelKind, error) { +func buildPanelKind(ctx context.Context, panelMap map[string]interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) (*dashv2alpha1.DashboardPanelKind, error) { panelID := float64(getIntField(panelMap, "id", 0)) // Transform queries - queries := transformPanelQueries(ctx, panelMap, dsInfoProvider) + queries := transformPanelQueries(ctx, panelMap, dsIndexProvider) // Transform transformations transformations := transformPanelTransformations(panelMap) @@ -864,7 +865,7 @@ func transformVariableSortToEnum(sort interface{}) dashv2alpha1.DashboardVariabl } } -func transformVariables(ctx context.Context, dashboard map[string]interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) ([]dashv2alpha1.DashboardVariableKind, error) { +func transformVariables(ctx context.Context, dashboard map[string]interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) ([]dashv2alpha1.DashboardVariableKind, error) { templating, ok := dashboard["templating"].(map[string]interface{}) if !ok { return []dashv2alpha1.DashboardVariableKind{}, nil @@ -889,11 +890,11 @@ func transformVariables(ctx context.Context, dashboard map[string]interface{}, d switch varType { case "query": - if queryVar, err := buildQueryVariable(ctx, varMap, commonProps, dsInfoProvider); err == nil { + if queryVar, err := buildQueryVariable(ctx, varMap, commonProps, dsIndexProvider); err == nil { variables = append(variables, queryVar) } case "datasource": - if dsVar, err := buildDatasourceVariable(ctx, varMap, commonProps, dsInfoProvider); err == nil { + if dsVar, err := buildDatasourceVariable(ctx, varMap, commonProps, dsIndexProvider); err == nil { variables = append(variables, dsVar) } case "custom": @@ -901,7 +902,7 @@ func transformVariables(ctx context.Context, dashboard map[string]interface{}, d variables = append(variables, customVar) } case "adhoc": - if adhocVar, err := buildAdhocVariable(ctx, varMap, commonProps, dsInfoProvider); err == nil { + if adhocVar, err := buildAdhocVariable(ctx, varMap, commonProps, dsIndexProvider); err == nil { variables = append(variables, adhocVar) } case "constant": @@ -917,7 +918,7 @@ func transformVariables(ctx context.Context, dashboard map[string]interface{}, d variables = append(variables, textVar) } case "groupby": - if groupByVar, err := buildGroupByVariable(ctx, varMap, commonProps, dsInfoProvider); err == nil { + if groupByVar, err := buildGroupByVariable(ctx, varMap, commonProps, dsIndexProvider); err == nil { variables = append(variables, groupByVar) } default: @@ -1093,7 +1094,7 @@ func buildDataQueryKindForVariable(query interface{}, datasourceType string) das } // Query Variable -func buildQueryVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsInfoProvider schemaversion.DataSourceInfoProvider) (dashv2alpha1.DashboardVariableKind, error) { +func buildQueryVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsIndexProvider schemaversion.DataSourceIndexProvider) (dashv2alpha1.DashboardVariableKind, error) { datasource := varMap["datasource"] var datasourceType, datasourceUID string @@ -1104,13 +1105,13 @@ func buildQueryVariable(ctx context.Context, varMap map[string]interface{}, comm // If we have a UID, use it to get the correct type from the datasource service // BUT: Don't try to resolve types for template variables if datasourceUID != "" && datasourceType == "" && !isTemplateVariable(datasourceUID) { - datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsInfoProvider) + datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsIndexProvider) } else if datasourceUID == "" && datasourceType == "" { // If no UID and no type, use default - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } } else { - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } queryVar := &dashv2alpha1.DashboardQueryVariableKind{ @@ -1161,8 +1162,8 @@ func buildQueryVariable(ctx context.Context, varMap map[string]interface{}, comm } // Datasource Variable -func buildDatasourceVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsInfoProvider schemaversion.DataSourceInfoProvider) (dashv2alpha1.DashboardVariableKind, error) { - pluginId := getDefaultDatasourceType(ctx, dsInfoProvider) +func buildDatasourceVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsIndexProvider schemaversion.DataSourceIndexProvider) (dashv2alpha1.DashboardVariableKind, error) { + pluginId := getDefaultDatasourceType(ctx, dsIndexProvider) if query := varMap["query"]; query != nil { if queryStr, ok := query.(string); ok { pluginId = queryStr @@ -1370,7 +1371,7 @@ func buildTextVariable(varMap map[string]interface{}, commonProps CommonVariable } // Adhoc Variable -func buildAdhocVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsInfoProvider schemaversion.DataSourceInfoProvider) (dashv2alpha1.DashboardVariableKind, error) { +func buildAdhocVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsIndexProvider schemaversion.DataSourceIndexProvider) (dashv2alpha1.DashboardVariableKind, error) { datasource := varMap["datasource"] var datasourceType, datasourceUID string @@ -1381,13 +1382,13 @@ func buildAdhocVariable(ctx context.Context, varMap map[string]interface{}, comm // If we have a UID, use it to get the correct type from the datasource service // BUT: Don't try to resolve types for template variables if datasourceUID != "" && datasourceType == "" && !isTemplateVariable(datasourceUID) { - datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsInfoProvider) + datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsIndexProvider) } else if datasourceUID == "" && datasourceType == "" { // If no UID and no type, use default - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } } else { - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } adhocVar := &dashv2alpha1.DashboardAdhocVariableKind{ @@ -1532,7 +1533,7 @@ func transformMetricFindValues(values []interface{}) []dashv2alpha1.DashboardMet } // GroupBy Variable -func buildGroupByVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsInfoProvider schemaversion.DataSourceInfoProvider) (dashv2alpha1.DashboardVariableKind, error) { +func buildGroupByVariable(ctx context.Context, varMap map[string]interface{}, commonProps CommonVariableProperties, dsIndexProvider schemaversion.DataSourceIndexProvider) (dashv2alpha1.DashboardVariableKind, error) { datasource := varMap["datasource"] var datasourceType, datasourceUID string @@ -1543,13 +1544,13 @@ func buildGroupByVariable(ctx context.Context, varMap map[string]interface{}, co // If we have a UID, use it to get the correct type from the datasource service // BUT: Don't try to resolve types for template variables if datasourceUID != "" && datasourceType == "" && !isTemplateVariable(datasourceUID) { - datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsInfoProvider) + datasourceType = getDatasourceTypeByUID(ctx, datasourceUID, dsIndexProvider) } else if datasourceUID == "" && datasourceType == "" { // If no UID and no type, use default - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } } else { - datasourceType = getDefaultDatasourceType(ctx, dsInfoProvider) + datasourceType = getDefaultDatasourceType(ctx, dsIndexProvider) } groupByVar := &dashv2alpha1.DashboardGroupByVariableKind{ @@ -1740,7 +1741,7 @@ func buildAnnotationFilter(filterMap map[string]interface{}) *dashv2alpha1.Dashb // Panel helper functions -func transformPanelQueries(ctx context.Context, panelMap map[string]interface{}, dsInfoProvider schemaversion.DataSourceInfoProvider) []dashv2alpha1.DashboardPanelQueryKind { +func transformPanelQueries(ctx context.Context, panelMap map[string]interface{}, dsIndexProvider schemaversion.DataSourceIndexProvider) []dashv2alpha1.DashboardPanelQueryKind { targets, ok := panelMap["targets"].([]interface{}) if !ok { return []dashv2alpha1.DashboardPanelQueryKind{} @@ -1755,10 +1756,10 @@ func transformPanelQueries(ctx context.Context, panelMap map[string]interface{}, // If we have a UID, use it to get the correct type from the datasource service // BUT: Don't try to resolve types for template variables if dsUID != "" && dsType == "" && !isTemplateVariable(dsUID) { - dsType = getDatasourceTypeByUID(ctx, dsUID, dsInfoProvider) + dsType = getDatasourceTypeByUID(ctx, dsUID, dsIndexProvider) } else if dsUID == "" && dsType == "" { // If no UID and no type, use default - dsType = getDefaultDatasourceType(ctx, dsInfoProvider) + dsType = getDefaultDatasourceType(ctx, dsIndexProvider) } panelDatasource = &dashv2alpha1.DashboardDataSourceRef{ @@ -1771,7 +1772,7 @@ func transformPanelQueries(ctx context.Context, panelMap map[string]interface{}, for _, target := range targets { if targetMap, ok := target.(map[string]interface{}); ok { - query := transformSingleQuery(ctx, targetMap, panelDatasource, dsInfoProvider) + query := transformSingleQuery(ctx, targetMap, panelDatasource, dsIndexProvider) queries = append(queries, query) } } @@ -1779,7 +1780,7 @@ func transformPanelQueries(ctx context.Context, panelMap map[string]interface{}, return queries } -func transformSingleQuery(ctx context.Context, targetMap map[string]interface{}, panelDatasource *dashv2alpha1.DashboardDataSourceRef, dsInfoProvider schemaversion.DataSourceInfoProvider) dashv2alpha1.DashboardPanelQueryKind { +func transformSingleQuery(ctx context.Context, targetMap map[string]interface{}, panelDatasource *dashv2alpha1.DashboardDataSourceRef, dsIndexProvider schemaversion.DataSourceIndexProvider) dashv2alpha1.DashboardPanelQueryKind { refId := schemaversion.GetStringValue(targetMap, "refId", "A") hidden := getBoolField(targetMap, "hide", false) @@ -1793,7 +1794,7 @@ func transformSingleQuery(ctx context.Context, targetMap map[string]interface{}, // If we have a UID, use it to get the correct type from the datasource service // BUT: Don't try to resolve types for template variables if queryDatasourceUID != "" && queryDatasourceType == "" && !isTemplateVariable(queryDatasourceUID) { - queryDatasourceType = getDatasourceTypeByUID(ctx, queryDatasourceUID, dsInfoProvider) + queryDatasourceType = getDatasourceTypeByUID(ctx, queryDatasourceUID, dsIndexProvider) } } else if panelDatasource != nil { // Only use panel datasource if it's not a mixed datasource diff --git a/apps/dashboard/pkg/migration/migrate.go b/apps/dashboard/pkg/migration/migrate.go index 9c887b553c6..cce80cfe209 100644 --- a/apps/dashboard/pkg/migration/migrate.go +++ b/apps/dashboard/pkg/migration/migrate.go @@ -9,24 +9,23 @@ import ( ) // Initialize provides the migrator singleton with required dependencies and builds the map of migrations. -func Initialize(dsInfoProvider schemaversion.DataSourceInfoProvider) { - migratorInstance.init(dsInfoProvider) +func Initialize(dsIndexProvider schemaversion.DataSourceIndexProvider) { + migratorInstance.init(dsIndexProvider) } -// GetDataSourceInfoProvider returns the datasource info provider instance that was initialized. -// This allows reuse of the same provider instance across migrations and conversions. -func GetDataSourceInfoProvider() schemaversion.DataSourceInfoProvider { +// GetDataSourceIndexProvider returns the datasource index provider instance that was initialized. +func GetDataSourceIndexProvider() schemaversion.DataSourceIndexProvider { // Wait for initialization to complete <-migratorInstance.ready - return migratorInstance.dsInfoProvider + return migratorInstance.dsIndexProvider } // ResetForTesting resets the migrator singleton for testing purposes. func ResetForTesting() { migratorInstance = &migrator{ - migrations: map[int]schemaversion.SchemaVersionMigrationFunc{}, - ready: make(chan struct{}), - dsInfoProvider: nil, + migrations: map[int]schemaversion.SchemaVersionMigrationFunc{}, + ready: make(chan struct{}), + dsIndexProvider: nil, } initOnce = sync.Once{} } @@ -46,15 +45,15 @@ var ( ) type migrator struct { - ready chan struct{} - migrations map[int]schemaversion.SchemaVersionMigrationFunc - dsInfoProvider schemaversion.DataSourceInfoProvider + ready chan struct{} + migrations map[int]schemaversion.SchemaVersionMigrationFunc + dsIndexProvider schemaversion.DataSourceIndexProvider } -func (m *migrator) init(dsInfoProvider schemaversion.DataSourceInfoProvider) { +func (m *migrator) init(dsIndexProvider schemaversion.DataSourceIndexProvider) { initOnce.Do(func() { - m.dsInfoProvider = dsInfoProvider - m.migrations = schemaversion.GetMigrations(dsInfoProvider) + m.dsIndexProvider = dsIndexProvider + m.migrations = schemaversion.GetMigrations(dsIndexProvider) close(m.ready) }) } diff --git a/apps/dashboard/pkg/migration/schemaversion/README.md b/apps/dashboard/pkg/migration/schemaversion/README.md index e1cfdc45a95..f1a5532e09a 100644 --- a/apps/dashboard/pkg/migration/schemaversion/README.md +++ b/apps/dashboard/pkg/migration/schemaversion/README.md @@ -96,7 +96,7 @@ func V{N}(_ context.Context, dashboard map[string]interface{}) error { Add your migration to `migrations.go`: ```go -func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionMigrationFunc { +func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc { return map[int]SchemaVersionMigrationFunc{ // ... existing migrations {N}: V{N}, // Add your migration here diff --git a/apps/dashboard/pkg/migration/schemaversion/datasource_utils.go b/apps/dashboard/pkg/migration/schemaversion/datasource_utils.go index 44d85cc20bf..b2020fa57cc 100644 --- a/apps/dashboard/pkg/migration/schemaversion/datasource_utils.go +++ b/apps/dashboard/pkg/migration/schemaversion/datasource_utils.go @@ -1,9 +1,110 @@ package schemaversion +import ( + "context" + "sync" +) + // Shared utility functions for datasource migrations across different schema versions. // These functions handle the common logic for migrating datasource references from // string names/UIDs to structured reference objects with uid, type, and apiVersion. +// onceIndexProvider wraps a DataSourceIndexProvider to ensure Index() is only called once. +// This prevents multiple DB queries and index builds during operations that may call +// provider.Index() multiple times (e.g., dashboard conversions with many datasource lookups). +// +// Thread-safe: Uses sync.Once to guarantee single execution even under concurrent access. +type onceIndexProvider struct { + provider DataSourceIndexProvider + once sync.Once + index *DatasourceIndex +} + +// Index returns the cached index, building it exactly once on first call. +func (p *onceIndexProvider) Index(ctx context.Context) *DatasourceIndex { + p.once.Do(func() { + p.index = p.provider.Index(ctx) + }) + return p.index +} + +// WrapIndexProviderWithOnce wraps a provider to cache the index for a single operation. +// Useful for conversions or migrations that may call provider.Index() multiple times. +// +// Example usage in dashboard conversion: +// +// onceDsIndexProvider := schemaversion.WrapIndexProviderWithOnce(dsIndexProvider) +// // Now all calls to onceDsIndexProvider.Index(ctx) return the same cached index +func WrapIndexProviderWithOnce(provider DataSourceIndexProvider) DataSourceIndexProvider { + if provider == nil { + return nil + } + return &onceIndexProvider{ + provider: provider, + } +} + +// DatasourceIndex provides O(1) lookup of datasources by name or UID. +type DatasourceIndex struct { + ByName map[string]*DataSourceInfo + ByUID map[string]*DataSourceInfo + DefaultDS *DataSourceInfo +} + +// NewDatasourceIndex creates an index from a list of datasources. +// Iterates once through the list to build name and UID maps for O(1) lookups. +func NewDatasourceIndex(datasources []DataSourceInfo) *DatasourceIndex { + idx := &DatasourceIndex{ + ByName: make(map[string]*DataSourceInfo, len(datasources)), + ByUID: make(map[string]*DataSourceInfo, len(datasources)), + } + + for i := range datasources { + ds := &datasources[i] + + // Index by name if present + if ds.Name != "" { + idx.ByName[ds.Name] = ds + } + + // Index by UID if present + if ds.UID != "" { + idx.ByUID[ds.UID] = ds + } + + // Track default datasource + if ds.Default { + idx.DefaultDS = ds + } + } + + return idx +} + +// Lookup finds a datasource by name or UID string. +// Returns the datasource info if found, nil otherwise. +func (idx *DatasourceIndex) Lookup(nameOrUID string) *DataSourceInfo { + // Try name first (most common in legacy dashboards) + if ds := idx.ByName[nameOrUID]; ds != nil { + return ds + } + // Try UID second + return idx.ByUID[nameOrUID] +} + +func (idx *DatasourceIndex) LookupByUID(uid string) *DataSourceInfo { + return idx.ByUID[uid] +} + +func (idx *DatasourceIndex) LookupByName(name string) *DataSourceInfo { + return idx.ByName[name] +} + +// GetDefault returns the default datasource, if one exists. +func (idx *DatasourceIndex) GetDefault() *DataSourceInfo { + return idx.DefaultDS +} + // GetDataSourceRef creates a datasource reference object with uid, type and optional apiVersion func GetDataSourceRef(ds *DataSourceInfo) map[string]interface{} { if ds == nil { @@ -19,21 +120,6 @@ func GetDataSourceRef(ds *DataSourceInfo) map[string]interface{} { return ref } -// GetDefaultDSInstanceSettings returns the default datasource if one exists -func GetDefaultDSInstanceSettings(datasources []DataSourceInfo) *DataSourceInfo { - for _, ds := range datasources { - if ds.Default { - return &DataSourceInfo{ - UID: ds.UID, - Type: ds.Type, - Name: ds.Name, - APIVersion: ds.APIVersion, - } - } - } - return nil -} - // isDataSourceRef checks if the object is a valid DataSourceRef (has uid or type) // Matches the frontend isDataSourceRef function in datasource.ts func isDataSourceRef(ref interface{}) bool { @@ -64,7 +150,7 @@ func isDataSourceRef(ref interface{}) bool { // Options: // - returnDefaultAsNull: if true, returns nil for "default" datasources (used in V33) // - returnDefaultAsNull: if false, returns reference for "default" datasources (used in V36) -func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, datasources []DataSourceInfo) map[string]interface{} { +func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, index *DatasourceIndex) map[string]interface{} { if options["returnDefaultAsNull"] && (nameOrRef == nil || nameOrRef == "default") { return nil } @@ -76,8 +162,7 @@ func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, // Look up datasource by name/UID if nameOrRef == nil || nameOrRef == "default" { - ds := GetDefaultDSInstanceSettings(datasources) - if ds != nil { + if ds := index.GetDefault(); ds != nil { return GetDataSourceRef(ds) } } @@ -90,16 +175,8 @@ func MigrateDatasourceNameToRef(nameOrRef interface{}, options map[string]bool, return map[string]interface{}{} } - // Search for matching datasource - for _, ds := range datasources { - if str == ds.Name || str == ds.UID { - return GetDataSourceRef(&DataSourceInfo{ - UID: ds.UID, - Type: ds.Type, - Name: ds.Name, - APIVersion: ds.APIVersion, - }) - } + if ds := index.Lookup(str); ds != nil { + return GetDataSourceRef(ds) } // Unknown datasource name should be preserved as UID-only reference diff --git a/apps/dashboard/pkg/migration/schemaversion/datasource_utils_test.go b/apps/dashboard/pkg/migration/schemaversion/datasource_utils_test.go index 85452a9c892..1be29a65c19 100644 --- a/apps/dashboard/pkg/migration/schemaversion/datasource_utils_test.go +++ b/apps/dashboard/pkg/migration/schemaversion/datasource_utils_test.go @@ -67,68 +67,13 @@ func TestGetDataSourceRef(t *testing.T) { } } -func TestGetDefaultDSInstanceSettings(t *testing.T) { - tests := []struct { - name string - datasources []schemaversion.DataSourceInfo - expected *schemaversion.DataSourceInfo - }{ - { - name: "empty datasources list", - datasources: []schemaversion.DataSourceInfo{}, - expected: nil, - }, - { - name: "no default datasource", - datasources: []schemaversion.DataSourceInfo{ - {UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false}, - {UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false}, - }, - expected: nil, - }, - { - name: "single default datasource", - datasources: []schemaversion.DataSourceInfo{ - {UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false}, - {UID: "default-ds-uid", Type: "prometheus", Name: "Default Test Datasource Name", Default: true, APIVersion: "v1"}, - {UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false}, - }, - expected: &schemaversion.DataSourceInfo{ - UID: "default-ds-uid", - Type: "prometheus", - Name: "Default Test Datasource Name", - APIVersion: "v1", - }, - }, - { - name: "multiple default datasources returns first", - datasources: []schemaversion.DataSourceInfo{ - {UID: "first-default", Type: "prometheus", Name: "First Default", Default: true, APIVersion: "v1"}, - {UID: "second-default", Type: "elasticsearch", Name: "Second Default", Default: true, APIVersion: "v2"}, - }, - expected: &schemaversion.DataSourceInfo{ - UID: "first-default", - Type: "prometheus", - Name: "First Default", - APIVersion: "v1", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := schemaversion.GetDefaultDSInstanceSettings(tt.datasources) - assert.Equal(t, tt.expected, result) - }) - } -} - func TestMigrateDatasourceNameToRef(t *testing.T) { datasources := []schemaversion.DataSourceInfo{ {UID: "default-ds-uid", Type: "prometheus", Name: "Default Test Datasource Name", Default: true, APIVersion: "v1"}, {UID: "existing-target-uid", Type: "elasticsearch", Name: "Existing Target Name", Default: false, APIVersion: "v2"}, {UID: "existing-ref-uid", Type: "prometheus", Name: "Existing Ref Name", Default: false, APIVersion: "v1"}, } + index := schemaversion.NewDatasourceIndex(datasources) t.Run("returnDefaultAsNull: true", func(t *testing.T) { options := map[string]bool{"returnDefaultAsNull": true} @@ -193,7 +138,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, datasources) + result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, index) assert.Equal(t, tt.expected, result) }) } @@ -261,7 +206,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, datasources) + result := schemaversion.MigrateDatasourceNameToRef(tt.nameOrRef, options, index) assert.Equal(t, tt.expected, result) }) } @@ -274,7 +219,7 @@ func TestMigrateDatasourceNameToRef(t *testing.T) { nameOrRef := map[string]interface{}{ "type": "prometheus", } - result := schemaversion.MigrateDatasourceNameToRef(nameOrRef, options, datasources) + result := schemaversion.MigrateDatasourceNameToRef(nameOrRef, options, index) expected := map[string]interface{}{ "type": "prometheus", } @@ -282,13 +227,14 @@ func TestMigrateDatasourceNameToRef(t *testing.T) { }) t.Run("integer input should return nil", func(t *testing.T) { - result := schemaversion.MigrateDatasourceNameToRef(123, options, datasources) + result := schemaversion.MigrateDatasourceNameToRef(123, options, index) expected := map[string]interface{}(nil) assert.Equal(t, expected, result) }) t.Run("empty datasources list", func(t *testing.T) { - result := schemaversion.MigrateDatasourceNameToRef("any-ds", options, []schemaversion.DataSourceInfo{}) + emptyIndex := schemaversion.NewDatasourceIndex([]schemaversion.DataSourceInfo{}) + result := schemaversion.MigrateDatasourceNameToRef("any-ds", options, emptyIndex) expected := map[string]interface{}{ "uid": "any-ds", } diff --git a/apps/dashboard/pkg/migration/schemaversion/migrations.go b/apps/dashboard/pkg/migration/schemaversion/migrations.go index a278c2e685e..9f670e751ef 100644 --- a/apps/dashboard/pkg/migration/schemaversion/migrations.go +++ b/apps/dashboard/pkg/migration/schemaversion/migrations.go @@ -22,10 +22,10 @@ type DataSourceInfo struct { APIVersion string } -type DataSourceInfoProvider interface { - // GetDataSourceInfo returns a list of all data sources with their info - // The context must have the namespace in it - GetDataSourceInfo(ctx context.Context) []DataSourceInfo +type DataSourceIndexProvider interface { + + // Index returns a pre-built index for O(1) datasource lookups. + Index(ctx context.Context) *DatasourceIndex } type PanelPluginInfo struct { @@ -33,7 +33,7 @@ type PanelPluginInfo struct { Version string } -func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionMigrationFunc { +func GetMigrations(dsIndexProvider DataSourceIndexProvider) map[int]SchemaVersionMigrationFunc { return map[int]SchemaVersionMigrationFunc{ 2: V2, 3: V3, @@ -66,10 +66,10 @@ func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionM 30: V30, 31: V31, 32: V32, - 33: V33(dsInfoProvider), + 33: V33(dsIndexProvider), 34: V34, 35: V35, - 36: V36(dsInfoProvider), + 36: V36(dsIndexProvider), 37: V37, 38: V38, 39: V39, diff --git a/apps/dashboard/pkg/migration/schemaversion/v33.go b/apps/dashboard/pkg/migration/schemaversion/v33.go index 738ee3118f2..05108854dfc 100644 --- a/apps/dashboard/pkg/migration/schemaversion/v33.go +++ b/apps/dashboard/pkg/migration/schemaversion/v33.go @@ -60,22 +60,24 @@ import ( // { "refId": "A", "datasource": "default" } // ] // } -func V33(dsInfo DataSourceInfoProvider) SchemaVersionMigrationFunc { +func V33(dsIndexProvider DataSourceIndexProvider) SchemaVersionMigrationFunc { return func(ctx context.Context, dashboard map[string]interface{}) error { - datasources := dsInfo.GetDataSourceInfo(ctx) if dashboard == nil { dashboard = map[string]interface{}{} } dashboard["schemaVersion"] = int(33) - migratePanelsV33(dashboard, datasources) + // Build datasource index directly from provider for O(1) lookups + index := dsIndexProvider.Index(ctx) + + migratePanelsV33(dashboard, index) return nil } } // migratePanelsV33 updates datasource references in dashboard panels for V33 migration -func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSourceInfo) { +func migratePanelsV33(dashboard map[string]interface{}, index *DatasourceIndex) { if dashboard == nil { return } @@ -90,7 +92,7 @@ func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSource continue } - migratePanelDatasourcesV33(panelMap, datasources) + migratePanelDatasourcesV33(panelMap, index) // Handle nested panels in collapsed rows nestedPanels, hasNested := panelMap["panels"].([]interface{}) @@ -103,15 +105,15 @@ func migratePanelsV33(dashboard map[string]interface{}, datasources []DataSource if !ok { continue } - migratePanelDatasourcesV33(np, datasources) + migratePanelDatasourcesV33(np, index) } } } // migratePanelDatasourcesV33 updates datasource references in a single panel and its targets for V33 migration -func migratePanelDatasourcesV33(panelMap map[string]interface{}, datasources []DataSourceInfo) { +func migratePanelDatasourcesV33(panelMap map[string]interface{}, index *DatasourceIndex) { // Handle panel datasource - always set result (even if nil) - if result := MigrateDatasourceNameToRef(panelMap["datasource"], map[string]bool{"returnDefaultAsNull": true}, datasources); result != nil { + if result := MigrateDatasourceNameToRef(panelMap["datasource"], map[string]bool{"returnDefaultAsNull": true}, index); result != nil { panelMap["datasource"] = result } else { panelMap["datasource"] = nil @@ -135,7 +137,7 @@ func migratePanelDatasourcesV33(panelMap map[string]interface{}, datasources []D } // Only set target datasource if migration result is not nil - if targetRef := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": true}, datasources); targetRef != nil { + if targetRef := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": true}, index); targetRef != nil { targetMap["datasource"] = targetRef } // If targetRef is nil, leave target.datasource unchanged (preserves "default" strings, etc.) diff --git a/apps/dashboard/pkg/migration/schemaversion/v33_test.go b/apps/dashboard/pkg/migration/schemaversion/v33_test.go index 7131fa8bab5..fa3ea433dcf 100644 --- a/apps/dashboard/pkg/migration/schemaversion/v33_test.go +++ b/apps/dashboard/pkg/migration/schemaversion/v33_test.go @@ -222,3 +222,7 @@ type testDataSourceProvider struct { func (p *testDataSourceProvider) GetDataSourceInfo(_ context.Context) []DataSourceInfo { return p.datasources } + +func (p *testDataSourceProvider) Index(_ context.Context) *DatasourceIndex { + return NewDatasourceIndex(p.datasources) +} diff --git a/apps/dashboard/pkg/migration/schemaversion/v36.go b/apps/dashboard/pkg/migration/schemaversion/v36.go index 7874e80b1d9..0a2c9e50e07 100644 --- a/apps/dashboard/pkg/migration/schemaversion/v36.go +++ b/apps/dashboard/pkg/migration/schemaversion/v36.go @@ -76,21 +76,21 @@ import ( // refId: "A" // }] // } -func V36(dsInfo DataSourceInfoProvider) SchemaVersionMigrationFunc { +func V36(dsIndexProvider DataSourceIndexProvider) SchemaVersionMigrationFunc { return func(ctx context.Context, dashboard map[string]interface{}) error { - datasources := dsInfo.GetDataSourceInfo(ctx) + dsIndex := dsIndexProvider.Index(ctx) dashboard["schemaVersion"] = int(36) - migrateAnnotations(dashboard, datasources) - migrateTemplateVariables(dashboard, datasources) - migratePanels(dashboard, datasources) + migrateAnnotations(dashboard, dsIndex) + migrateTemplateVariables(dashboard, dsIndex) + migratePanels(dashboard, dsIndex) return nil } } // migrateAnnotations updates datasource references in dashboard annotations -func migrateAnnotations(dashboard map[string]interface{}, datasources []DataSourceInfo) { +func migrateAnnotations(dashboard map[string]interface{}, index *DatasourceIndex) { annotations, ok := dashboard["annotations"].(map[string]interface{}) if !ok { return @@ -109,12 +109,12 @@ func migrateAnnotations(dashboard map[string]interface{}, datasources []DataSour // Always migrate datasource, even if it doesn't exist (will be set to default) ds := queryMap["datasource"] - queryMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources) + queryMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index) } } // migrateTemplateVariables updates datasource references in dashboard variables -func migrateTemplateVariables(dashboard map[string]interface{}, datasources []DataSourceInfo) { +func migrateTemplateVariables(dashboard map[string]interface{}, index *DatasourceIndex) { templating, ok := dashboard["templating"].(map[string]interface{}) if !ok { return @@ -125,7 +125,7 @@ func migrateTemplateVariables(dashboard map[string]interface{}, datasources []Da return } - defaultDS := GetDefaultDSInstanceSettings(datasources) + defaultDS := index.GetDefault() for _, variable := range list { varMap, ok := variable.(map[string]interface{}) if !ok { @@ -149,7 +149,7 @@ func migrateTemplateVariables(dashboard map[string]interface{}, datasources []Da } // migratePanels updates datasource references in dashboard panels -func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInfo) { +func migratePanels(dashboard map[string]interface{}, index *DatasourceIndex) { panels, ok := dashboard["panels"].([]interface{}) if !ok { return @@ -160,7 +160,7 @@ func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInf if !ok { continue } - migratePanelDatasources(panelMap, datasources) + migratePanelDatasources(panelMap, index) // Handle nested panels in collapsed rows nestedPanels, hasNested := panelMap["panels"].([]interface{}) @@ -173,24 +173,24 @@ func migratePanels(dashboard map[string]interface{}, datasources []DataSourceInf if !ok { continue } - migratePanelDatasourcesInternal(np, datasources, true) + migratePanelDatasourcesInternal(np, index, true) } } } // migratePanelDatasources updates datasource references in a single panel and its targets -func migratePanelDatasources(panelMap map[string]interface{}, datasources []DataSourceInfo) { - migratePanelDatasourcesInternal(panelMap, datasources, false) +func migratePanelDatasources(panelMap map[string]interface{}, index *DatasourceIndex) { + migratePanelDatasourcesInternal(panelMap, index, false) } // migratePanelDatasourcesInternal updates datasource references with nesting awareness -func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasources []DataSourceInfo, isNested bool) { +func migratePanelDatasourcesInternal(panelMap map[string]interface{}, index *DatasourceIndex, isNested bool) { // NOTE: Even though row panels don't technically need datasource or targets fields, // we process them anyway to exactly match frontend behavior and avoid inconsistencies // between frontend and backend migrations. The frontend DashboardMigrator processes // all panels uniformly without special row panel handling. - defaultDS := GetDefaultDSInstanceSettings(datasources) + defaultDS := index.GetDefault() panelDataSourceWasDefault := false // Handle targets - only add default targets to top-level panels (matches frontend behavior) @@ -228,7 +228,7 @@ func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasource // Keep empty object {} as-is (set by V33 migration for empty strings) panelMap["datasource"] = ds } else { - migrated := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources) + migrated := MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index) panelMap["datasource"] = migrated } } @@ -273,12 +273,12 @@ func migratePanelDatasourcesInternal(panelMap map[string]interface{}, datasource targetMap["datasource"] = result } else { // Frontend: target.datasource = migrateDatasourceNameToRef(target.datasource, { returnDefaultAsNull: false }); - targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources) + targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index) } } } else { // Migrate existing target datasource - targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, datasources) + targetMap["datasource"] = MigrateDatasourceNameToRef(ds, map[string]bool{"returnDefaultAsNull": false}, index) } // Update panel datasource if it was default and target is not an expression diff --git a/apps/dashboard/pkg/migration/testutil/mocks.go b/apps/dashboard/pkg/migration/testutil/mocks.go index dbc7df72728..49c1df0f403 100644 --- a/apps/dashboard/pkg/migration/testutil/mocks.go +++ b/apps/dashboard/pkg/migration/testutil/mocks.go @@ -37,6 +37,12 @@ func (p *ConfigurableDataSourceProvider) GetDataSourceInfo(_ context.Context) [] } } +// Index builds the index directly from the datasources +func (p *ConfigurableDataSourceProvider) Index(ctx context.Context) *schemaversion.DatasourceIndex { + datasources := p.GetDataSourceInfo(ctx) + return schemaversion.NewDatasourceIndex(datasources) +} + // getStandardTestDataSources returns datasources for standard migration tests func (p *ConfigurableDataSourceProvider) getStandardTestDataSources() []schemaversion.DataSourceInfo { return []schemaversion.DataSourceInfo{ diff --git a/pkg/registry/apis/dashboard/datasources.go b/pkg/registry/apis/dashboard/datasources.go index 891938009f9..c3cb18efdcc 100644 --- a/pkg/registry/apis/dashboard/datasources.go +++ b/pkg/registry/apis/dashboard/datasources.go @@ -8,41 +8,69 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" ) -type datasourceInfoProvider struct { +type datasourceIndexProvider struct { datasourceService datasources.DataSourceService } -func (d *datasourceInfoProvider) GetDataSourceInfo(ctx context.Context) []schemaversion.DataSourceInfo { +// Index builds a datasource index directly from the datasource service query. +// This is more efficient than GetDataSourceInfo + NewDatasourceIndex as it avoids +// creating an intermediate slice and iterates over the datasources only once. +func (d *datasourceIndexProvider) Index(ctx context.Context) *schemaversion.DatasourceIndex { // Extract namespace info from context to get OrgID nsInfo, err := request.NamespaceInfoFrom(ctx, true) if err != nil { - // If namespace info is not available, return empty list - return []schemaversion.DataSourceInfo{} + // If namespace info is not available, return empty index + return &schemaversion.DatasourceIndex{ + ByName: make(map[string]*schemaversion.DataSourceInfo), + ByUID: make(map[string]*schemaversion.DataSourceInfo), + } } - // Use GetDataSources with OrgID query instead of GetAllDataSources - // This ensures tenant-aware datasource retrieval + // Use GetDataSources with OrgID query query := datasources.GetDataSourcesQuery{ OrgID: nsInfo.OrgID, } dataSources, err := d.datasourceService.GetDataSources(ctx, &query) if err != nil { - return []schemaversion.DataSourceInfo{} + return &schemaversion.DatasourceIndex{ + ByName: make(map[string]*schemaversion.DataSourceInfo), + ByUID: make(map[string]*schemaversion.DataSourceInfo), + } } - out := make([]schemaversion.DataSourceInfo, 0, len(dataSources)) + // Build index directly without intermediate slice allocation + // Single iteration over datasources populates all maps + index := &schemaversion.DatasourceIndex{ + ByName: make(map[string]*schemaversion.DataSourceInfo, len(dataSources)), + ByUID: make(map[string]*schemaversion.DataSourceInfo, len(dataSources)), + } for _, ds := range dataSources { - out = append(out, schemaversion.DataSourceInfo{ + dsInfo := &schemaversion.DataSourceInfo{ Name: ds.Name, UID: ds.UID, ID: ds.ID, Type: ds.Type, Default: ds.IsDefault, APIVersion: ds.APIVersion, - }) + } + + // Index by name if present + if ds.Name != "" { + index.ByName[ds.Name] = dsInfo + } + + // Index by UID if present + if ds.UID != "" { + index.ByUID[ds.UID] = dsInfo + } + + // Track default datasource + if ds.IsDefault { + index.DefaultDS = dsInfo + } } - return out + return index } diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 20629866d6f..1cbd289ddaa 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -175,14 +175,14 @@ func RegisterAPIService( } migration.RegisterMetrics(reg) - migration.Initialize(&datasourceInfoProvider{ + migration.Initialize(&datasourceIndexProvider{ datasourceService: datasourceService, }) apiregistration.RegisterAPI(builder) return builder } -func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceInfoProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder { +func NewAPIService(ac authlib.AccessClient, features featuremgmt.FeatureToggles, folderClientProvider client.K8sHandlerProvider, datasourceProvider schemaversion.DataSourceIndexProvider, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *DashboardsAPIBuilder { migration.Initialize(datasourceProvider) return &DashboardsAPIBuilder{ minRefreshInterval: "10s", @@ -231,7 +231,7 @@ func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { } // Register the explicit conversions - if err := conversion.RegisterConversions(scheme, migration.GetDataSourceInfoProvider()); err != nil { + if err := conversion.RegisterConversions(scheme, migration.GetDataSourceIndexProvider()); err != nil { return err }