diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 90565ef3eb0..138e05f187e 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -302,6 +302,10 @@ export interface FeatureToggles { */ dashboardSchemaValidationLogging?: boolean; /** + * Enable fallback parsing behavior when scan row encounters invalid dashboard JSON + */ + scanRowInvalidDashboardParseFallbackEnabled?: boolean; + /** * Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) */ datasourceQueryTypes?: boolean; diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go index 8e46423be87..89053e6b932 100644 --- a/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go +++ b/pkg/cmd/grafana-cli/commands/datamigrations/to_unified_storage.go @@ -63,6 +63,12 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err }, } + featureManager, err := featuremgmt.ProvideManagerService(cfg) + if err != nil { + return err + } + featureToggles := featuremgmt.ProvideToggles(featureManager) + provisioning, err := newStubProvisioning(cfg.ProvisioningPath) if err != nil { return err @@ -76,9 +82,10 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err nil, // no librarypanels.Service sort.ProvideService(), acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), + featureToggles, ) - client, err := newUnifiedClient(cfg, sqlStore) + client, err := newUnifiedClient(cfg, sqlStore, featureToggles) if err != nil { return err } @@ -215,12 +222,7 @@ func promptYesNo(prompt string) (bool, error) { } } -func newUnifiedClient(cfg *setting.Cfg, sqlStore db.DB) (resource.ResourceClient, error) { - featureManager, err := featuremgmt.ProvideManagerService(cfg) - if err != nil { - return nil, err - } - featureToggles := featuremgmt.ProvideToggles(featureManager) +func newUnifiedClient(cfg *setting.Cfg, sqlStore db.DB, featureToggles featuremgmt.FeatureToggles) (resource.ResourceClient, error) { return unified.ProvideUnifiedStorageClient(&unified.Options{ Cfg: cfg, Features: featureToggles, diff --git a/pkg/registry/apis/dashboard/legacy/migrate.go b/pkg/registry/apis/dashboard/legacy/migrate.go index 3da3a01aab8..73d01caaade 100644 --- a/pkg/registry/apis/dashboard/legacy/migrate.go +++ b/pkg/registry/apis/dashboard/legacy/migrate.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/librarypanels" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/search/sort" @@ -51,9 +52,10 @@ func ProvideLegacyMigrator( provisioning provisioning.ProvisioningService, // only needed for dashboard settings libraryPanelSvc librarypanels.Service, accessControl accesscontrol.AccessControl, + features featuremgmt.FeatureToggles, ) LegacyMigrator { dbp := legacysql.NewDatabaseProvider(sql) - return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), accessControl) + return NewDashboardAccess(dbp, authlib.OrgNamespaceFormatter, nil, provisioning, libraryPanelSvc, sort.ProvideService(), accessControl, features) } type BlobStoreInfo struct { @@ -309,11 +311,12 @@ func (a *dashboardSqlAccess) migrateDashboards(ctx context.Context, orgId int64, for _, row := range rows.rejected { id := row.Dash.Labels[utils.LabelKeyDeprecatedInternalID] a.log.Warn("rejected dashboard", + "namespace", opts.Namespace, "dashboard", row.Dash.Name, "uid", row.Dash.UID, "id", id, + "version", row.Dash.Generation, "stackId", opts.StackID, - "namespace", opts.Namespace, ) opts.Progress(-2, fmt.Sprintf("rejected: id:%s, uid:%s", id, row.Dash.Name)) } diff --git a/pkg/registry/apis/dashboard/legacy/query_dashboards.sql b/pkg/registry/apis/dashboard/legacy/query_dashboards.sql index 75439c60d03..e8503170a2c 100644 --- a/pkg/registry/apis/dashboard/legacy/query_dashboards.sql +++ b/pkg/registry/apis/dashboard/legacy/query_dashboards.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go index 0771f0fc3ee..f019a39a799 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go @@ -29,6 +29,7 @@ import ( "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/librarypanels" "github.com/grafana/grafana/pkg/services/provisioning" @@ -62,6 +63,8 @@ type dashboardSqlAccess struct { namespacer request.NamespaceMapper provisioning provisioning.ProvisioningService + invalidDashboardParseFallbackEnabled bool + // Use for writing (not reading) dashStore dashboards.Store dashboardSearchClient legacysearcher.DashboardSearchClient @@ -82,17 +85,19 @@ func NewDashboardAccess(sql legacysql.LegacyDatabaseProvider, libraryPanelSvc librarypanels.Service, sorter sort.Service, accessControl accesscontrol.AccessControl, + features featuremgmt.FeatureToggles, ) DashboardAccess { dashboardSearchClient := legacysearcher.NewDashboardSearchClient(dashStore, sorter) return &dashboardSqlAccess{ - sql: sql, - namespacer: namespacer, - dashStore: dashStore, - provisioning: provisioning, - dashboardSearchClient: *dashboardSearchClient, - libraryPanelSvc: libraryPanelSvc, - accessControl: accessControl, - log: log.New("dashboard.legacysql"), + sql: sql, + namespacer: namespacer, + dashStore: dashStore, + provisioning: provisioning, + dashboardSearchClient: *dashboardSearchClient, + libraryPanelSvc: libraryPanelSvc, + accessControl: accessControl, + log: log.New("dashboard.legacysql"), + invalidDashboardParseFallbackEnabled: features.IsEnabled(context.Background(), featuremgmt.FlagScanRowInvalidDashboardParseFallbackEnabled), } } @@ -176,7 +181,7 @@ func (r *rowsWrapper) Next() bool { r.row, err = r.a.scanRow(r.rows, r.history) if err != nil { r.a.log.Error("error scanning dashboard", "error", err) - if len(r.rejected) > 0 || r.row == nil { + if len(r.rejected) > 100 || r.row == nil { r.err = fmt.Errorf("too many rejected rows (%d) %w", len(r.rejected), err) return false } @@ -228,6 +233,51 @@ func (r *rowsWrapper) Value() []byte { return b } +func generateFallbackDashboard(data []byte, title, uid string) ([]byte, error) { + generatedDashboard := map[string]interface{}{ + "editable": true, + "id": 1, + "panels": []map[string]interface{}{ + { + "description": "The JSON is invalid. You can import it again after fixing it.", + "gridPos": map[string]interface{}{"h": 8, "w": 24, "x": 0, "y": 0}, + "id": 1, + "options": map[string]interface{}{ + "code": map[string]interface{}{"language": "plaintext", "showLineNumbers": false, "showMiniMap": false}, + "content": string(data), + "mode": "code", + }, + "title": "Invalid dashboard", + "type": "text", + }, + }, + "schemaVersion": 41, + "title": title, + "uid": uid, + "version": 3, + } + return json.Marshal(generatedDashboard) +} + +func (a *dashboardSqlAccess) parseDashboard(dash *dashboardV1.Dashboard, data []byte, id int64, title string) error { + if err := dash.Spec.UnmarshalJSON(data); err != nil { + a.log.Warn("error unmarshalling dashboard spec. Generating fallback dashboard data", "error", err, "uid", dash.UID, "name", dash.Name) + dash.Spec = *dashboardV0.NewDashboardSpec() + + dashboardData, err := generateFallbackDashboard(data, title, string(dash.UID)) + if err != nil { + a.log.Warn("error generating fallback dashboard data", "error", err, "uid", dash.UID, "name", dash.Name) + return err + } + + if err = dash.Spec.UnmarshalJSON(dashboardData); err != nil { + a.log.Warn("error unmarshalling fallback dashboard data", "error", err, "uid", dash.UID, "name", dash.Name) + return err + } + } + return nil +} + func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRow, error) { dash := &dashboardV1.Dashboard{ TypeMeta: dashboardV1.DashboardResourceInfo.TypeMeta(), @@ -238,6 +288,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo var dashboard_id int64 var orgId int64 var folder_uid sql.NullString + var title string var updated time.Time var updatedBy sql.NullString var updatedByID sql.NullInt64 @@ -257,7 +308,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo var data []byte // the dashboard JSON var version int64 - err := rows.Scan(&orgId, &dashboard_id, &dash.Name, &folder_uid, + err := rows.Scan(&orgId, &dashboard_id, &dash.Name, &title, &folder_uid, &deleted, &plugin_id, &origin_name, &origin_path, &origin_hash, &origin_ts, &created, &createdBy, &createdByID, @@ -286,6 +337,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo dash.SetCreationTimestamp(metav1.NewTime(created)) meta, err := utils.MetaAccessor(dash) if err != nil { + a.log.Debug("failed to get meta accessor for dashboard", "error", err, "uid", dash.UID, "name", dash.Name, "version", version) return nil, err } meta.SetUpdatedTimestamp(&updated) @@ -331,9 +383,14 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo } if len(data) > 0 { - err = dash.Spec.UnmarshalJSON(data) - if err != nil { - return row, fmt.Errorf("JSON unmarshal error for: %s // %w", dash.Name, err) + if a.invalidDashboardParseFallbackEnabled { + if err := a.parseDashboard(dash, data, dashboard_id, title); err != nil { + return row, err + } + } else { + if err := dash.Spec.UnmarshalJSON(data); err != nil { + return row, fmt.Errorf("JSON unmarshal error for: %s // %w", dash.Name, err) + } } } // Ignore any saved values for id/version/uid diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go index 41da615b6ff..da8d2bfe25d 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards_test.go @@ -32,13 +32,15 @@ func TestScanRow(t *testing.T) { provisioner := provisioning.NewProvisioningServiceMock(context.Background()) provisioner.GetDashboardProvisionerResolvedPathFunc = func(name string) string { return "provisioner" } store := &dashboardSqlAccess{ - namespacer: func(_ int64) string { return "default" }, - provisioning: provisioner, - log: log.New("test"), + namespacer: func(_ int64) string { return "default" }, + provisioning: provisioner, + log: log.New("test"), + invalidDashboardParseFallbackEnabled: false, } - columns := []string{"orgId", "dashboard_id", "name", "folder_uid", "deleted", "plugin_id", "origin_name", "origin_path", "origin_hash", "origin_ts", "created", "createdBy", "createdByID", "updated", "updatedBy", "updatedByID", "version", "message", "data", "api_version"} + columns := []string{"orgId", "dashboard_id", "name", "title", "folder_uid", "deleted", "plugin_id", "origin_name", "origin_path", "origin_hash", "origin_ts", "created", "createdBy", "createdByID", "updated", "updatedBy", "updatedByID", "version", "message", "data", "api_version"} id := int64(100) + uid := "someuid" title := "Test Dashboard" folderUID := "folder123" timestamp := time.Now() @@ -49,7 +51,7 @@ func TestScanRow(t *testing.T) { updatedUser := "updator" t.Run("Should scan a valid row correctly", func(t *testing.T) { - rows := sqlmock.NewRows(columns).AddRow(1, id, title, folderUID, nil, "", "", "", "", 0, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") + rows := sqlmock.NewRows(columns).AddRow(1, id, uid, title, folderUID, nil, "", "", "", "", 0, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") mock.ExpectQuery("SELECT *").WillReturnRows(rows) resultRows, err := mockDB.Query("SELECT *") require.NoError(t, err) @@ -59,7 +61,7 @@ func TestScanRow(t *testing.T) { row, err := store.scanRow(resultRows, false) require.NoError(t, err) require.NotNil(t, row) - require.Equal(t, "Test Dashboard", row.Dash.Name) + require.Equal(t, uid, row.Dash.Name) require.Equal(t, version, row.RV) // rv should be the dashboard version require.Equal(t, common.Unstructured{ Object: map[string]interface{}{"key": "value"}, @@ -80,7 +82,7 @@ func TestScanRow(t *testing.T) { }) t.Run("File provisioned dashboard should have annotations", func(t *testing.T) { - rows := sqlmock.NewRows(columns).AddRow(1, id, title, folderUID, nil, "", "provisioner", pathToFile, "hashing", 100000, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") + rows := sqlmock.NewRows(columns).AddRow(1, id, uid, title, folderUID, nil, "", "provisioner", pathToFile, "hashing", 100000, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") mock.ExpectQuery("SELECT *").WillReturnRows(rows) resultRows, err := mockDB.Query("SELECT *") require.NoError(t, err) @@ -108,7 +110,7 @@ func TestScanRow(t *testing.T) { }) t.Run("Plugin provisioned dashboard should have annotations", func(t *testing.T) { - rows := sqlmock.NewRows(columns).AddRow(1, id, title, folderUID, nil, "slo", "", "", "", 0, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") + rows := sqlmock.NewRows(columns).AddRow(1, id, uid, title, folderUID, nil, "slo", "", "", "", 0, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, []byte(`{"key": "value"}`), "vXyz") mock.ExpectQuery("SELECT *").WillReturnRows(rows) resultRows, err := mockDB.Query("SELECT *") require.NoError(t, err) @@ -144,7 +146,7 @@ func TestScanRow(t *testing.T) { // In migration scenario, COALESCE functions return dashboard table values // when dashboard_version values are NULL, ensuring all dashboards are migrated rows := sqlmock.NewRows(columns).AddRow( - 1, id, title, folderUID, nil, "", // basic dashboard fields + 1, id, uid, title, folderUID, nil, "", // basic dashboard fields "", "", "", 0, // origin fields timestamp, createdUser, 0, // created fields // These represent COALESCED values from dashboard table (not version table) @@ -163,7 +165,8 @@ func TestScanRow(t *testing.T) { require.NotNil(t, row) // Verify migration scenario works correctly with fallback data - require.Equal(t, title, row.Dash.Name) + require.Equal(t, uid, row.Dash.Name) + require.Equal(t, "Migrated Dashboard", row.Dash.Spec.Object["title"]) require.Equal(t, migrationVersion, row.RV) // Should use COALESCEd dashboard table version require.Equal(t, common.Unstructured{ Object: map[string]interface{}{ @@ -187,6 +190,72 @@ func TestScanRow(t *testing.T) { require.Equal(t, folderUID, meta.GetFolder()) require.Equal(t, "dashboard.grafana.app/"+migrationAPIVersion, row.Dash.APIVersion) }) + + t.Run("should follow dashboard template when failing to unmarshal dashboard if feature flag X is enabled", func(t *testing.T) { + // row with bad data + badData := []byte(`{"rows":[{"panels":[{"targets":[{"refId":"A","target":"aliasSub(alias, '^(.{27}).+', '\1...')"}]}]}]}`) + rows := sqlmock.NewRows(columns).AddRow(1, id, uid, title, folderUID, nil, "", "", "", "", 0, timestamp, createdUser, 0, timestamp, updatedUser, 0, version, message, badData, "vXyz") + mock.ExpectQuery("SELECT *").WillReturnRows(rows) + resultRows, err := mockDB.Query("SELECT *") + require.NoError(t, err) + defer resultRows.Close() // nolint:errcheck + resultRows.Next() + + row, err := store.scanRow(resultRows, false) + require.Error(t, err, "JSON unmarshal error for: Test Dashboard // invalid character '1' in string escape code") + require.NotNil(t, row) + // correctly scans these + require.Equal(t, uid, row.Dash.Name) + require.Equal(t, version, row.RV) + require.Equal(t, "default", row.Dash.Namespace) + require.Equal(t, &continueToken{orgId: int64(1), id: id}, row.token) + + // failure case: does NOT parse the dashboard itself + require.Equal(t, common.Unstructured{ + Object: nil, + }, row.Dash.Spec) + + // store with feature flag enabled + store = &dashboardSqlAccess{ + namespacer: func(_ int64) string { return "default" }, + provisioning: provisioner, + log: log.New("test"), + invalidDashboardParseFallbackEnabled: true, + } + + row, err = store.scanRow(resultRows, false) + require.NoError(t, err) + require.NotNil(t, row) + require.Equal(t, uid, row.Dash.Name) + require.Equal(t, version, row.RV) + require.Equal(t, "default", row.Dash.Namespace) + require.Equal(t, &continueToken{orgId: int64(1), id: id}, row.token) + + // instead of failing, create dummy dashboard with broken json inlined in text panel + require.Equal(t, title, row.Dash.Spec.Object["title"]) + panels, exists := row.Dash.Spec.Object["panels"] + require.True(t, exists, "panels property should exist") + + panelsSlice, ok := panels.([]interface{}) + require.True(t, ok, "panels should be a slice") + require.Len(t, panelsSlice, 1, "panels should have exactly one element") + + panel, ok := panelsSlice[0].(map[string]interface{}) + require.True(t, ok, "panel should be a map") + + options, exists := panel["options"] + require.True(t, exists, "panel should have options property") + + optionsMap, ok := options.(map[string]interface{}) + require.True(t, ok, "options should be a map") + + content, exists := optionsMap["content"] + require.True(t, exists, "options should have content property") + + contentStr, ok := content.(string) + require.True(t, ok, "content should be a string") + require.Equal(t, string(badData), contentStr, "content should match bad json data") + }) } func TestBuildSaveDashboardCommand(t *testing.T) { diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard.sql index e3a945a53db..63f1231205e 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_next_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_next_page.sql index 862725168b0..0992bbd3ce2 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_next_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-dashboard_next_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql index fd54e6dc949..1d57a86d450 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-export_with_history.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-folders.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-folders.sql index 5935eb3422c..38cf8f56bea 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-folders.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-folders.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid.sql index ec0a40a5934..5c1a6974590 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_at_version.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_at_version.sql index e04b1ff7430..bf85c6ba3d5 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_at_version.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_at_version.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_second_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_second_page.sql index ec0a40a5934..5c1a6974590 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_second_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-history_uid_second_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-migration_with_fallback.sql b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-migration_with_fallback.sql index ba38c048136..8c3801ba771 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-migration_with_fallback.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/mysql--query_dashboards-migration_with_fallback.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard.sql index 552a486f8cd..c04123f90ab 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_next_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_next_page.sql index fb09d57d08d..5916a9b315f 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_next_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-dashboard_next_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql index 4d0affb337f..5eec54770c7 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-export_with_history.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-folders.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-folders.sql index b994d617708..5c9cca1e30e 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-folders.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-folders.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid.sql index 9bcbb168149..876fca02fe3 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_at_version.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_at_version.sql index 61311c5c99b..0fa23f2db08 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_at_version.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_at_version.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_second_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_second_page.sql index 9bcbb168149..876fca02fe3 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_second_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-history_uid_second_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-migration_with_fallback.sql b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-migration_with_fallback.sql index 364fbebf417..fee2c28d525 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-migration_with_fallback.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/postgres--query_dashboards-migration_with_fallback.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard.sql index 552a486f8cd..c04123f90ab 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_next_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_next_page.sql index fb09d57d08d..5916a9b315f 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_next_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-dashboard_next_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql index 4d0affb337f..5eec54770c7 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-export_with_history.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-folders.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-folders.sql index b994d617708..5c9cca1e30e 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-folders.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-folders.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid.sql index 9bcbb168149..876fca02fe3 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_at_version.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_at_version.sql index 61311c5c99b..0fa23f2db08 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_at_version.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_at_version.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_second_page.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_second_page.sql index 9bcbb168149..876fca02fe3 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_second_page.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-history_uid_second_page.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-migration_with_fallback.sql b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-migration_with_fallback.sql index 364fbebf417..fee2c28d525 100755 --- a/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-migration_with_fallback.sql +++ b/pkg/registry/apis/dashboard/legacy/testdata/sqlite--query_dashboards-migration_with_fallback.sql @@ -2,6 +2,7 @@ SELECT dashboard.org_id, dashboard.id, dashboard.uid, + dashboard.title, dashboard.folder_uid, dashboard.deleted, plugin_id, diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 19108edadf8..75e2ed5b518 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -143,7 +143,7 @@ func RegisterAPIService( folderClient: folderClient, legacy: &DashboardStorage{ - Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, accessControl), + Access: legacy.NewDashboardAccess(dbp, namespacer, dashStore, provisioning, libraryPanelSvc, sorter, accessControl, features), DashboardService: dashboardService, }, reg: reg, diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index b398b85d5a9..aeff3aa1924 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -746,7 +746,7 @@ func Initialize(cfg *setting.Cfg, opts Options, apiOpts api.ServerOptions) (*Ser } userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer) factory := github.ProvideFactory() - legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl) + legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl, featureToggles) databaseDatabase := database5.ProvideDatabase(sqlStore, tracer) secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(databaseDatabase, tracer, registerer) if err != nil { @@ -1307,7 +1307,7 @@ func InitializeForTest(t sqlutil.ITestDB, testingT interface { } userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer) factory := github.ProvideFactory() - legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl) + legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl, featureToggles) databaseDatabase := database5.ProvideDatabase(sqlStore, tracer) secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(databaseDatabase, tracer, registerer) if err != nil { diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 0ac3c4d9ce2..f50b7854ed6 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -501,6 +501,12 @@ var ( Stage: FeatureStageExperimental, Owner: grafanaAppPlatformSquad, }, + { + Name: "scanRowInvalidDashboardParseFallbackEnabled", + Description: "Enable fallback parsing behavior when scan row encounters invalid dashboard JSON", + Stage: FeatureStageExperimental, + Owner: grafanaSearchAndStorageSquad, + }, { Name: "datasourceQueryTypes", Description: "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 7b5c27e5123..6c21f852785 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -65,6 +65,7 @@ kubernetesDashboards,experimental,@grafana/grafana-app-platform-squad,false,fals dashboardDisableSchemaValidationV1,experimental,@grafana/grafana-app-platform-squad,false,false,false dashboardDisableSchemaValidationV2,experimental,@grafana/grafana-app-platform-squad,false,false,false dashboardSchemaValidationLogging,experimental,@grafana/grafana-app-platform-squad,false,false,false +scanRowInvalidDashboardParseFallbackEnabled,experimental,@grafana/search-and-storage,false,false,false datasourceQueryTypes,experimental,@grafana/grafana-app-platform-squad,false,true,false queryService,experimental,@grafana/grafana-datasources-core-services,false,true,false queryServiceRewrite,experimental,@grafana/grafana-datasources-core-services,false,true,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index b1cc2fb7f13..52552a0b8d9 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -271,6 +271,10 @@ const ( // Log schema validation errors so they can be analyzed later FlagDashboardSchemaValidationLogging = "dashboardSchemaValidationLogging" + // FlagScanRowInvalidDashboardParseFallbackEnabled + // Enable fallback parsing behavior when scan row encounters invalid dashboard JSON + FlagScanRowInvalidDashboardParseFallbackEnabled = "scanRowInvalidDashboardParseFallbackEnabled" + // FlagDatasourceQueryTypes // Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) FlagDatasourceQueryTypes = "datasourceQueryTypes" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 048f5512581..980f805d56d 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -2760,6 +2760,18 @@ "codeowner": "@grafana/identity-access-team" } }, + { + "metadata": { + "name": "scanRowInvalidDashboardParseFallbackEnabled", + "resourceVersion": "1753730899886", + "creationTimestamp": "2025-07-28T19:28:19Z" + }, + "spec": { + "description": "Enable fallback parsing behavior when scan row encounters invalid dashboard JSON", + "stage": "experimental", + "codeowner": "@grafana/search-and-storage" + } + }, { "metadata": { "name": "scopeApi",