From 7356df32b98b5b4031a04433b9691c9c1db03c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Dec 2025 15:05:10 +0100 Subject: [PATCH] [release-11.6.10] Fix tables without PKs (including tables from current main) (#115537) * Skip empty migrations. * Backport SecretDB to release-11.6, and initialize it on startup. * Add missing file. * Move dependency registration to apis.go * Use migrator from Grafana 12.2. * Add migrations introducing cloud_migration_snapshot_partition table. * Fix import. * Convert unique keys in 3 tables to primary keys (#115421) * Added method for adding migrations for convering unique to primary key. Based on existing migration for `file` table (in `db_file_storage.go`) migrations. * Added better default migration names. Added ability to override migration name. * Use ConvertUniqueKeyToPrimaryKey for cloud_migration_snapshot_partition table. * Convert resource_version UQE to PK. * Convert secret_encrypted_value UQE to PK. * Removed extra test. * Removed testdata. * Remove support for renaming migrations for now. We can bring it in later, when we want to convert existing migrations for file, file_meta and setting tables. * Revert removal of ColumnName to ease backporting, since this field is referenced from enterprise code. * Use quoted identifiers in Postgres statement. * Ignore data_key_id column that was introduced in Grafana 12.3. * Return empty SQL, this is now ignored. --- pkg/registry/apis/apis.go | 2 + pkg/registry/apis/secret/contracts/keeper.go | 6 + .../apis/secret/contracts/migrator.go | 8 + pkg/registry/apis/secret/register.go | 24 ++ pkg/registry/apis/wireset.go | 4 + pkg/server/wire.go | 3 + pkg/server/wire_gen.go | 18 +- .../sqlstore/migrations/cloud_migrations.go | 28 +++ .../sqlstore/migrations/db_file_storage.go | 4 +- pkg/services/sqlstore/migrator/dialect.go | 2 +- pkg/services/sqlstore/migrator/migrations.go | 156 +++++++++++- .../sqlstore/migrator/migrations_test.go | 79 ++++++ pkg/services/sqlstore/migrator/migrator.go | 15 +- .../sqlite_file_migration_statement.sql | 23 ++ pkg/services/sqlstore/migrator/testing.go | 51 ++++ pkg/storage/secret/migrator/migrator.go | 224 ++++++++++++++++++ .../unified/sql/db/migrations/resource_mig.go | 12 + 17 files changed, 649 insertions(+), 10 deletions(-) create mode 100644 pkg/registry/apis/secret/contracts/keeper.go create mode 100644 pkg/registry/apis/secret/contracts/migrator.go create mode 100644 pkg/registry/apis/secret/register.go create mode 100644 pkg/services/sqlstore/migrator/migrations_test.go create mode 100644 pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql create mode 100644 pkg/services/sqlstore/migrator/testing.go create mode 100644 pkg/storage/secret/migrator/migrator.go diff --git a/pkg/registry/apis/apis.go b/pkg/registry/apis/apis.go index aebcfcbbdd0..2038d952289 100644 --- a/pkg/registry/apis/apis.go +++ b/pkg/registry/apis/apis.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/iam" "github.com/grafana/grafana/pkg/registry/apis/provisioning" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/secret" "github.com/grafana/grafana/pkg/registry/apis/userstorage" ) @@ -28,6 +29,7 @@ func ProvideRegistryServiceSink( _ *notifications.NotificationsAPIBuilder, _ *userstorage.UserStorageAPIBuilder, _ *provisioning.APIBuilder, + _ *secret.DependencyRegisterer, ) *Service { return &Service{} } diff --git a/pkg/registry/apis/secret/contracts/keeper.go b/pkg/registry/apis/secret/contracts/keeper.go new file mode 100644 index 00000000000..987ef4318e7 --- /dev/null +++ b/pkg/registry/apis/secret/contracts/keeper.go @@ -0,0 +1,6 @@ +package contracts + +var ( + // The name used to refer to the system keeper + SystemKeeperName = "system" +) diff --git a/pkg/registry/apis/secret/contracts/migrator.go b/pkg/registry/apis/secret/contracts/migrator.go new file mode 100644 index 00000000000..c804c8dee24 --- /dev/null +++ b/pkg/registry/apis/secret/contracts/migrator.go @@ -0,0 +1,8 @@ +package contracts + +import "context" + +// SecretDBMigrator is an interface for running database migrations related to secrets management. +type SecretDBMigrator interface { + RunMigrations(ctx context.Context, lockDatabase bool) error +} diff --git a/pkg/registry/apis/secret/register.go b/pkg/registry/apis/secret/register.go new file mode 100644 index 00000000000..7661574a07d --- /dev/null +++ b/pkg/registry/apis/secret/register.go @@ -0,0 +1,24 @@ +package secret + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" + "github.com/grafana/grafana/pkg/setting" +) + +// DependencyRegisterer is set to satisfy wire gen and make sure the `RegisterDependencies` is called. +type DependencyRegisterer struct{} + +func RegisterDependencies(cfg *setting.Cfg, secretDBMigrator contracts.SecretDBMigrator) (*DependencyRegisterer, error) { + // Some DBs that claim to be MySQL/Postgres-compatible might not support table locking. + lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true) + + // This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator. + if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil { + return nil, fmt.Errorf("running secret database migrations: %w", err) + } + + return &DependencyRegisterer{}, nil +} diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index ecd0cba0afb..d4ba0b02763 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/iam" "github.com/grafana/grafana/pkg/registry/apis/provisioning" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/secret" "github.com/grafana/grafana/pkg/registry/apis/service" "github.com/grafana/grafana/pkg/registry/apis/userstorage" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" @@ -25,6 +26,9 @@ var WireSet = wire.NewSet( wire.Bind(new(datasource.PluginContextWrapper), new(*plugincontext.Provider)), datasource.ProvideDefaultPluginConfigs, + // Secrets + secret.RegisterDependencies, + // Each must be added here *and* in the ServiceSink above dashboardinternal.RegisterAPIService, dashboardsnapshot.RegisterAPIService, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 82b7e891669..b7b536a4200 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -10,6 +10,7 @@ import ( "github.com/google/wire" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/api/avatar" "github.com/grafana/grafana/pkg/api/routing" @@ -158,6 +159,7 @@ import ( "github.com/grafana/grafana/pkg/services/user/userimpl" "github.com/grafana/grafana/pkg/setting" legacydualwrite "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + secretmigrator "github.com/grafana/grafana/pkg/storage/secret/migrator" "github.com/grafana/grafana/pkg/storage/unified/resource" unifiedsearch "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/tsdb/azuremonitor" @@ -401,6 +403,7 @@ var wireBasicSet = wire.NewSet( connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, + secretmigrator.NewWithEngine, // Unified storage resource.ProvideStorageMetrics, // Kubernetes API server diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index c7e943c6a30..499b5f90199 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -56,6 +56,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/iam" provisioning2 "github.com/grafana/grafana/pkg/registry/apis/provisioning" query2 "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/secret" "github.com/grafana/grafana/pkg/registry/apis/userstorage" "github.com/grafana/grafana/pkg/registry/apps" "github.com/grafana/grafana/pkg/registry/apps/advisor" @@ -221,6 +222,7 @@ import ( "github.com/grafana/grafana/pkg/services/validations" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + migrator2 "github.com/grafana/grafana/pkg/storage/secret/migrator" "github.com/grafana/grafana/pkg/storage/unified" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/search" @@ -709,7 +711,12 @@ func Initialize(cfg *setting.Cfg, opts Options, apiOpts api.ServerOptions) (*Ser if err != nil { return nil, err } - apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, notificationsAPIBuilder, userStorageAPIBuilder, apiBuilder) + secretDBMigrator := migrator2.NewWithEngine(sqlStore) + dependencyRegisterer, err := secret.RegisterDependencies(cfg, secretDBMigrator) + if err != nil { + return nil, err + } + apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, notificationsAPIBuilder, userStorageAPIBuilder, apiBuilder, dependencyRegisterer) teamPermissionsService, err := ossaccesscontrol.ProvideTeamPermissions(cfg, featureToggles, routeRegisterImpl, sqlStore, accessControl, ossLicensingService, acimplService, teamService, userService, actionSetService) if err != nil { return nil, err @@ -1194,7 +1201,12 @@ func InitializeForTest(t sqlutil.ITestDB, cfg *setting.Cfg, opts Options, apiOpt if err != nil { return nil, err } - apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, notificationsAPIBuilder, userStorageAPIBuilder, apiBuilder) + secretDBMigrator := migrator2.NewWithEngine(sqlStore) + dependencyRegisterer, err := secret.RegisterDependencies(cfg, secretDBMigrator) + if err != nil { + return nil, err + } + apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, notificationsAPIBuilder, userStorageAPIBuilder, apiBuilder, dependencyRegisterer) teamPermissionsService, err := ossaccesscontrol.ProvideTeamPermissions(cfg, featureToggles, routeRegisterImpl, sqlStore, accessControl, ossLicensingService, acimplService, teamService, userService, actionSetService) if err != nil { return nil, err @@ -1345,7 +1357,7 @@ func InitializeDocumentBuilders(cfg *setting.Cfg) (resource.DocumentBuilderSuppl // wire.go: -var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatechecker.ProvideGrafanaService, updatechecker.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator2.ProvideService, legacy.ProvideLegacyMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, pushhttp.ProvideService, contexthandler.ProvideService, service10.ProvideService, wire.Bind(new(service10.LDAP), new(*service10.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service5.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service5.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database4.DashboardSnapshotStore)), database4.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service8.ServiceImpl)), service8.ProvideService, service5.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service5.Service)), service5.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager2.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, featuremgmt.ProvideOpenFeatureService, service6.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service6.DashboardServiceImpl)), service6.ProvideDashboardService, service6.ProvideDashboardProvisioningService, service6.ProvideDashboardPluginService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), folderimpl.ProvideDashboardFolderStore, wire.Bind(new(folder.FolderStore), new(*folderimpl.DashboardFolderStoreImpl)), service9.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service9.ImportDashboardService)), service7.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service7.Service)), service7.ProvideDashboardUpdater, guardian2.ProvideService, sanitizer.ProvideService, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideService, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, interceptors.ProvideAuthenticator, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations2.ProvideDataSourceMigrationService, migrations2.ProvideMigrateToPluginService, migrations2.ProvideMigrateFromPluginService, migrations2.ProvideSecretMigrationProvider, wire.Bind(new(migrations2.SecretMigrationProvider), new(*migrations2.SecretMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, resource.ProvideStorageMetrics, apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet) +var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatechecker.ProvideGrafanaService, updatechecker.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator2.ProvideService, legacy.ProvideLegacyMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, pushhttp.ProvideService, contexthandler.ProvideService, service10.ProvideService, wire.Bind(new(service10.LDAP), new(*service10.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service5.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service5.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database4.DashboardSnapshotStore)), database4.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service8.ServiceImpl)), service8.ProvideService, service5.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service5.Service)), service5.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager2.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, featuremgmt.ProvideOpenFeatureService, service6.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service6.DashboardServiceImpl)), service6.ProvideDashboardService, service6.ProvideDashboardProvisioningService, service6.ProvideDashboardPluginService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), folderimpl.ProvideDashboardFolderStore, wire.Bind(new(folder.FolderStore), new(*folderimpl.DashboardFolderStoreImpl)), service9.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service9.ImportDashboardService)), service7.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service7.Service)), service7.ProvideDashboardUpdater, guardian2.ProvideService, sanitizer.ProvideService, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideService, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, interceptors.ProvideAuthenticator, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations2.ProvideDataSourceMigrationService, migrations2.ProvideMigrateToPluginService, migrations2.ProvideMigrateFromPluginService, migrations2.ProvideSecretMigrationProvider, wire.Bind(new(migrations2.SecretMigrationProvider), new(*migrations2.SecretMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, migrator2.NewWithEngine, resource.ProvideStorageMetrics, apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet) var wireSet = wire.NewSet( wireBasicSet, metrics.WireSet, sqlstore.ProvideService, metrics2.ProvideService, wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), wire.Bind(new(notifications.WebhookSender), new(*notifications.NotificationService)), wire.Bind(new(notifications.EmailSender), new(*notifications.NotificationService)), wire.Bind(new(db.DB), new(*sqlstore.SQLStore)), prefimpl.ProvideService, oauthtoken.ProvideService, wire.Bind(new(oauthtoken.OAuthTokenService), new(*oauthtoken.Service)), diff --git a/pkg/services/sqlstore/migrations/cloud_migrations.go b/pkg/services/sqlstore/migrations/cloud_migrations.go index 18dc3590570..90338fe8d50 100644 --- a/pkg/services/sqlstore/migrations/cloud_migrations.go +++ b/pkg/services/sqlstore/migrations/cloud_migrations.go @@ -98,6 +98,15 @@ func addCloudMigrationsMigrations(mg *Migrator) { {Cols: []string{"uid"}, Type: UniqueIndex}, }, } + migrationSnapshotPartitionTable := Table{ + Name: "cloud_migration_snapshot_partition", + Columns: []*Column{ + {Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false}, + {Name: "partition_number", Type: DB_Int, Nullable: false}, + {Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false}, + {Name: "data", Type: DB_LongBlob, Nullable: false}, + }, + } addTableReplaceMigrations(mg, migrationTable, migrationSessionTable, 2, map[string]string{ "id": "id", @@ -187,4 +196,23 @@ func addCloudMigrationsMigrations(mg *Migrator) { mg.AddMigration("increase resource_uid column length", NewRawSQLMigration(""). Mysql("ALTER TABLE cloud_migration_resource MODIFY resource_uid NVARCHAR(255);"). Postgres("ALTER TABLE cloud_migration_resource ALTER COLUMN resource_uid TYPE VARCHAR(255);")) + + mg.AddMigration("create cloud_migration_snapshot_partition table v1", NewAddTableMigration(migrationSnapshotPartitionTable)) + srpUniqueIndex := Index{ + Name: "srp_unique", + Cols: []string{"snapshot_uid", "resource_type", "partition_number"}, Type: UniqueIndex, + } + mg.AddMigration("add cloud_migration_snapshot_partition srp_unique index", NewAddIndexMigration(migrationSnapshotPartitionTable, &srpUniqueIndex)) + + updatedCloudMigrationSnapshotPartitionTable := Table{ + Name: "cloud_migration_snapshot_partition", + Columns: []*Column{ + {Name: "snapshot_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true}, + {Name: "partition_number", Type: DB_Int, Nullable: false, IsPrimaryKey: true}, + {Name: "resource_type", Type: DB_Varchar, Length: 255, Nullable: false, IsPrimaryKey: true}, + {Name: "data", Type: DB_LongBlob, Nullable: false}, + }, + PrimaryKeys: []string{"snapshot_uid", "resource_type", "partition_number"}, + } + ConvertUniqueKeyToPrimaryKey(mg, srpUniqueIndex, updatedCloudMigrationSnapshotPartitionTable) } diff --git a/pkg/services/sqlstore/migrations/db_file_storage.go b/pkg/services/sqlstore/migrations/db_file_storage.go index e8ca9348455..c44344307ac 100644 --- a/pkg/services/sqlstore/migrations/db_file_storage.go +++ b/pkg/services/sqlstore/migrations/db_file_storage.go @@ -91,7 +91,7 @@ func convertFilePathHashIndexToPrimaryKey(mg *migrator.Migrator) { mg.AddMigration("drop file_path unique index from file table if it exists (mysql)", mysqlMigration2) mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file ADD PRIMARY KEY (path_hash);`) - mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file", ColumnName: "path_hash"} + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file"} mg.AddMigration("add primary key to file table if it doesn't exist (mysql)", mysqlMigration3) postgres := ` @@ -162,7 +162,7 @@ func convertFileMetaPathHashKeyIndexToPrimaryKey(mg *migrator.Migrator) { mg.AddMigration("drop file_path unique index from file_meta table if it exists (mysql)", mysqlMigration2) mysqlMigration3 := migrator.NewRawSQLMigration("").Mysql(`ALTER TABLE file_meta ADD PRIMARY KEY (path_hash, ` + "`key`" + `);`) - mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta", ColumnName: "path_hash"} + mysqlMigration3.Condition = &migrator.IfPrimaryKeyNotExistsCondition{TableName: "file_meta"} mg.AddMigration("add primary key to file_meta table if it doesn't exist (mysql)", mysqlMigration3) postgres := ` diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index 92ed26156fa..3bf854d323c 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -241,7 +241,7 @@ func (b *BaseDialect) CopyTableData(sourceTable string, targetTable string, sour targetColsSQL := b.QuoteColList(targetCols) quote := b.dialect.Quote - return fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable)) + return fmt.Sprintf("INSERT INTO %s (%s)\nSELECT %s\nFROM %s", quote(targetTable), targetColsSQL, sourceColsSQL, quote(sourceTable)) } func (b *BaseDialect) DropTable(tableName string) string { diff --git a/pkg/services/sqlstore/migrator/migrations.go b/pkg/services/sqlstore/migrator/migrations.go index 025c2b36d32..89a754d3756 100644 --- a/pkg/services/sqlstore/migrator/migrations.go +++ b/pkg/services/sqlstore/migrator/migrations.go @@ -1,6 +1,8 @@ package migrator import ( + "fmt" + "slices" "strings" ) @@ -53,7 +55,7 @@ func (m *RawSQLMigration) SQL(dialect Dialect) string { } } - return dialect.NoOpSQL() + return "" } func (m *RawSQLMigration) Set(dialect string, sql string) *RawSQLMigration { @@ -275,3 +277,155 @@ func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharset func (m *TableCharsetMigration) SQL(d Dialect) string { return d.UpdateTableSQL(m.tableName, m.columns) } + +type addPrimaryKeyMigration struct { + MigrationBase + tableName string + uniqueKey Index + + // Used for Sqlite recreation of the table. Temporary table will have tableName + "_new" suffix. + table Table +} + +func (m *addPrimaryKeyMigration) SQL(d Dialect) string { + if d.DriverName() == SQLite { + // Final SQL will do following in the individual statements: + // 1. Create new temporary table + // 2. Copy data from old table to temporary table + // 3. Drop old table, rename temporary table to original name + // 4. Recreate indexes for table. + // + // For example: + // + // CREATE TABLE file_new + // ( + // path TEXT NOT NULL, + // path_hash TEXT NOT NULL, + // parent_folder_path_hash TEXT NOT NULL, + // contents BLOB NOT NULL, + // etag TEXT NOT NULL, + // cache_control TEXT NOT NULL, + // content_disposition TEXT NOT NULL, + // updated DATETIME NOT NULL, + // created DATETIME NOT NULL, + // size INTEGER NOT NULL, + // mime_type TEXT NOT NULL, + // + // PRIMARY KEY (path_hash) + // ); + // + // INSERT INTO file_new (path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type) + // SELECT path, path_hash, parent_folder_path_hash, contents, etag, cache_control, content_disposition, updated, created, size, mime_type FROM file; + // + // DROP TABLE file; + // ALTER TABLE file_new RENAME TO file; + // + // CREATE INDEX IDX_file_parent_folder_path_hash ON file (parent_folder_path_hash); + + tempTable := m.table + tempTable.Name = m.tableName + "_new" + + statements := strings.Builder{} + + statements.WriteString(d.CreateTableSQL(&tempTable)) + statements.WriteString("\n") // CreateTableSQL adds semicolon + + cols := make([]string, 0, len(tempTable.Columns)) + for _, col := range tempTable.Columns { + cols = append(cols, col.Name) + } + statements.WriteString(d.CopyTableData(m.tableName, tempTable.Name, cols, cols)) + statements.WriteString(";\n") + + statements.WriteString(d.DropTable(m.tableName)) + statements.WriteString(";\n") + + statements.WriteString(d.RenameTable(tempTable.Name, m.tableName)) + statements.WriteString(";\n") + + for _, idx := range tempTable.Indices { + // Use real table name, not temporary one now + statements.WriteString(d.CreateIndexSQL(m.tableName, idx)) + statements.WriteString("\n") // CreateIndexSQL adds semicolon + } + + return statements.String() + } else if d.DriverName() == Postgres { + quotesCols := make([]string, 0, len(m.uniqueKey.Cols)) + for _, c := range m.uniqueKey.Cols { + quotesCols = append(quotesCols, d.Quote(c)) + } + + return fmt.Sprintf(` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS %s; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = '%s'::regclass AND indisprimary) THEN + ALTER TABLE %s ADD PRIMARY KEY (%s); + END IF; + END $$;`, d.Quote(m.uniqueKey.XName(m.tableName)), m.tableName, d.Quote(m.tableName), strings.Join(quotesCols, ",")) + } else { + return "" + } +} + +// ConvertUniqueKeyToPrimaryKey adds series of migrations to convert existing unique key to PRIMARY KEY. +// For Sqlite this means recreating the table, which only works if there are no foreign keys referencing the table. +func ConvertUniqueKeyToPrimaryKey(mg *Migrator, uniqueKey Index, finalTable Table) { + tableName := finalTable.Name + if tableName == "" { + panic("invalid table name") + } + if len(uniqueKey.Cols) == 0 || uniqueKey.Type != UniqueIndex { + panic("invalid unique type") + } + if !slices.Equal(uniqueKey.Cols, finalTable.PrimaryKeys) { + panic("invalid primary key in the final table") + } + + colPks := map[string]bool{} + for _, col := range finalTable.Columns { + if col.IsPrimaryKey { + colPks[col.Name] = true + } + } + for _, c := range uniqueKey.Cols { + if !colPks[c] { + panic(fmt.Sprintf("column %s is not part of primary key in the table definition", c)) + } + } + + columnsList := strings.Join(uniqueKey.Cols, ",") + + mysqlQuote := NewDialect(MySQL).Quote + mysqlQuotedColumns := make([]string, 0, len(uniqueKey.Cols)) + for _, col := range uniqueKey.Cols { + mysqlQuotedColumns = append(mysqlQuotedColumns, mysqlQuote(col)) + } + + // migration 1 is to handle cases where the table was created with sql_generate_invisible_primary_key = ON + // in this case we need to do the conversion in one sql statement + mysqlMigration1 := NewRawSQLMigration("").Mysql(fmt.Sprintf(` + ALTER TABLE %s + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX %s, + ADD PRIMARY KEY (%s); + `, tableName, uniqueKey.XName(tableName), strings.Join(mysqlQuotedColumns, ","))) + mysqlMigration1.Condition = &IfColumnExistsCondition{TableName: tableName, ColumnName: "my_row_id"} + mg.AddMigration(fmt.Sprintf("drop my_row_id and add primary key with columns %s to table %s if my_row_id exists (auto-generated mysql column)", columnsList, tableName), mysqlMigration1) + + mysqlMigration2 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s DROP INDEX %s`, tableName, uniqueKey.XName(tableName))) + mysqlMigration2.Condition = &IfIndexExistsCondition{TableName: tableName, IndexName: uniqueKey.XName(tableName)} + mg.AddMigration(fmt.Sprintf("drop unique index %s from %s table if it exists (mysql)", uniqueKey.XName(tableName), tableName), mysqlMigration2) + + mysqlMigration3 := NewRawSQLMigration("").Mysql(fmt.Sprintf(`ALTER TABLE %s ADD PRIMARY KEY (%s)`, tableName, strings.Join(mysqlQuotedColumns, ","))) + mysqlMigration3.Condition = &IfPrimaryKeyNotExistsCondition{TableName: tableName} + mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s if it doesn't exist (mysql)", columnsList, tableName), mysqlMigration3) + + // postgres and sqlite statements are idempotent so we can have only one condition-less migration + mg.AddMigration(fmt.Sprintf("add primary key with columns %s to table %s (postgres and sqlite)", columnsList, tableName), &addPrimaryKeyMigration{tableName: tableName, uniqueKey: uniqueKey, table: finalTable}) +} diff --git a/pkg/services/sqlstore/migrator/migrations_test.go b/pkg/services/sqlstore/migrator/migrations_test.go new file mode 100644 index 00000000000..ebf7c4f48f6 --- /dev/null +++ b/pkg/services/sqlstore/migrator/migrations_test.go @@ -0,0 +1,79 @@ +package migrator + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/require" +) + +//go:embed testdata/sqlite_file_migration_statement.sql +var sqliteMigrationStatement string + +func TestConvertUniqueKeyToPrimaryKey(t *testing.T) { + names := []string{ + "drop my_row_id and add primary key with columns path_hash,etag to table file if my_row_id exists (auto-generated mysql column)", + "drop unique index UQE_file_path_hash_etag from file table if it exists (mysql)", + "add primary key with columns path_hash,etag to table file if it doesn't exist (mysql)", + "add primary key with columns path_hash,etag to table file (postgres and sqlite)", + } + expectedMigrations := map[string][]ExpectedMigration{ + MySQL: { + {Id: names[0], SQL: ` + ALTER TABLE file + DROP PRIMARY KEY, + DROP COLUMN my_row_id, + DROP INDEX UQE_file_path_hash_etag, + ADD PRIMARY KEY (` + "`path_hash`" + `,` + "`etag`" + `);`}, + {Id: names[1], SQL: "ALTER TABLE file DROP INDEX UQE_file_path_hash_etag"}, + {Id: names[2], SQL: "ALTER TABLE file ADD PRIMARY KEY (`path_hash`,`etag`)"}, + {Id: names[3], SQL: ""}, + }, + Postgres: { + {Id: names[0], SQL: ""}, + {Id: names[1], SQL: ""}, + {Id: names[2], SQL: ""}, + {Id: names[3], SQL: ` + DO $$ + BEGIN + -- Drop the unique constraint if it exists + DROP INDEX IF EXISTS "UQE_file_path_hash_etag"; + + -- Add primary key if it doesn't already exist + IF NOT EXISTS (SELECT 1 FROM pg_index i WHERE indrelid = 'file'::regclass AND indisprimary) THEN + ALTER TABLE "file" ADD PRIMARY KEY ("path_hash","etag"); + END IF; + END $$;`}, + }, + SQLite: { + {Id: names[0], SQL: ""}, + {Id: names[1], SQL: ""}, + {Id: names[2], SQL: ""}, + {Id: names[3], SQL: sqliteMigrationStatement}, // Embed used here because sqlite statement is full of backquotes. + }, + } + + for dialectName, migrations := range expectedMigrations { + t.Run(dialectName, func(t *testing.T) { + err := CheckExpectedMigrations(dialectName, migrations, func(migrator *Migrator) { + ConvertUniqueKeyToPrimaryKey(migrator, + Index{Cols: []string{"path_hash", "etag"}, Type: UniqueIndex}, // Convert this unique key to primary key + Table{ + Name: "file", + Columns: []*Column{ + {Name: "path", Type: DB_NVarchar, Length: 1024, Nullable: false}, + {Name: "path_hash", Type: DB_NVarchar, Length: 64, Nullable: false, IsPrimaryKey: true}, + {Name: "parent_folder_path_hash", Type: DB_NVarchar, Length: 64, Nullable: false}, + {Name: "contents", Type: DB_Blob, Nullable: false}, + {Name: "etag", Type: DB_NVarchar, Length: 32, Nullable: false, IsPrimaryKey: true}, + }, + PrimaryKeys: []string{"path_hash", "etag"}, + Indices: []*Index{ + {Cols: []string{"parent_folder_path_hash"}}, + }, + }) + }) + require.NoError(t, err) + }) + } +} diff --git a/pkg/services/sqlstore/migrator/migrator.go b/pkg/services/sqlstore/migrator/migrator.go index b970711a80e..f2fad097c82 100644 --- a/pkg/services/sqlstore/migrator/migrator.go +++ b/pkg/services/sqlstore/migrator/migrator.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" _ "github.com/go-sql-driver/mysql" @@ -67,12 +68,16 @@ func NewMigrator(engine *xorm.Engine, cfg *setting.Cfg) *Migrator { // NewScopedMigrator should only be used for the transition to a new storage engine func NewScopedMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string) *Migrator { + return newMigrator(engine, cfg, scope, NewDialect(engine.DriverName())) +} + +func newMigrator(engine *xorm.Engine, cfg *setting.Cfg, scope string, dialect Dialect) *Migrator { mg := &Migrator{ Cfg: cfg, DBEngine: engine, migrations: make([]Migration, 0), migrationIds: make(map[string]struct{}), - Dialect: NewDialect(engine.DriverName()), + Dialect: dialect, metrics: migratorMetrics{ migCount: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "grafana_database", @@ -392,8 +397,12 @@ func (mg *Migrator) exec(ctx context.Context, m Migration, sess *xorm.Session) e err = codeMigration.Exec(sess, mg) } else { sql := m.SQL(mg.Dialect) - logger.Debug("Executing sql migration", "id", m.Id(), "sql", sql) - _, err = sess.Exec(sql) + if strings.TrimSpace(sql) == "" { + logger.Debug("Skipping empty sql migration", "id", m.Id()) + } else { + logger.Debug("Executing sql migration", "id", m.Id(), "sql", sql) + _, err = sess.Exec(sql) + } } if err != nil { diff --git a/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql b/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql new file mode 100644 index 00000000000..2f37839d842 --- /dev/null +++ b/pkg/services/sqlstore/migrator/testdata/sqlite_file_migration_statement.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS `file_new` ( + `path` TEXT NOT NULL + , `path_hash` TEXT NOT NULL + , `parent_folder_path_hash` TEXT NOT NULL + , `contents` BLOB NOT NULL + , `etag` TEXT NOT NULL + , PRIMARY KEY ( `path_hash`,`etag` )); + +INSERT INTO `file_new` (`path` + , `path_hash` + , `parent_folder_path_hash` + , `contents` + , `etag`) +SELECT `path` + , `path_hash` + , `parent_folder_path_hash` + , `contents` + , `etag` +FROM `file`; + +DROP TABLE IF EXISTS `file`; +ALTER TABLE `file_new` RENAME TO `file`; +CREATE INDEX `IDX_file_parent_folder_path_hash` ON `file` (`parent_folder_path_hash`); diff --git a/pkg/services/sqlstore/migrator/testing.go b/pkg/services/sqlstore/migrator/testing.go new file mode 100644 index 00000000000..cfe7849c79e --- /dev/null +++ b/pkg/services/sqlstore/migrator/testing.go @@ -0,0 +1,51 @@ +package migrator + +import ( + "fmt" + "strings" +) + +type ExpectedMigration struct { + Id string + SQL string +} + +// CheckExpectedMigrations verifies that given migrations exist in migrator after running addMigrations function, +// that they are in the same order and have expected SQL. +func CheckExpectedMigrations(dialectName string, expected []ExpectedMigration, addMigrations func(migrator *Migrator)) error { + d := NewDialect(dialectName) + mg := newMigrator(nil, nil, "", d) + addMigrations(mg) + + migrations := mg.migrations + migrationNames := make([]string, 0, len(migrations)) + for _, m := range expected { + for ; len(migrations) > 0 && migrations[0].Id() != m.Id; migrations = migrations[1:] { + migrationNames = append(migrationNames, migrations[0].Id()) + } + + if len(migrations) == 0 { + return fmt.Errorf("migration `%s` not found, existing migrations:\n%s", m.Id, strings.Join(migrationNames, "\n")) + } + + sql := migrations[0].SQL(d) + if normalizeLines(m.SQL) != normalizeLines(sql) { + return fmt.Errorf("migration `%s` has wrong SQL:\nexpected:\n%s\nactual:\n%s", m.Id, m.SQL, sql) + } + } + return nil +} + +func normalizeLines(sql string) string { + lines := strings.Split(sql, "\n") + result := strings.Builder{} + for _, l := range lines { + l := strings.TrimSpace(l) + if l == "" { + continue + } + result.WriteString(l) + result.WriteString("\n") + } + return result.String() +} diff --git a/pkg/storage/secret/migrator/migrator.go b/pkg/storage/secret/migrator/migrator.go new file mode 100644 index 00000000000..8c1aa8a2c3b --- /dev/null +++ b/pkg/storage/secret/migrator/migrator.go @@ -0,0 +1,224 @@ +package migrator + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "xorm.io/xorm" +) + +const ( + TableNameKeeper = "secret_keeper" + TableNameSecureValue = "secret_secure_value" + TableNameDataKey = "secret_data_key" + TableNameEncryptedValue = "secret_encrypted_value" +) + +type SecretDB struct { + engine *xorm.Engine +} + +func New() registry.DatabaseMigrator { + return &SecretDB{} +} + +func NewWithEngine(db db.DB) contracts.SecretDBMigrator { + return &SecretDB{engine: db.GetEngine()} +} + +func (db *SecretDB) RunMigrations(ctx context.Context, lockDatabase bool) error { + mg := migrator.NewScopedMigrator(db.engine, nil, "secret") + + db.AddMigration(mg) + + return mg.RunMigrations(ctx, lockDatabase, 0) +} + +func (*SecretDB) AddMigration(mg *migrator.Migrator) { + mg.AddCreateMigration() + + mg.AddMigration("Initialize secrets tables", &migrator.RawSQLMigration{}) + + tables := []migrator.Table{} + + secureValueTable := migrator.Table{ + Name: TableNameSecureValue, + Columns: []*migrator.Column{ + // Kubernetes Metadata + {Name: "guid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "annotations", Type: migrator.DB_Text, Nullable: true}, + {Name: "labels", Type: migrator.DB_Text, Nullable: true}, + {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "created_by", Type: migrator.DB_Text, Nullable: false}, + {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, // Used as RV (ResourceVersion) + {Name: "updated_by", Type: migrator.DB_Text, Nullable: false}, + + // Kubernetes Status + {Name: "external_id", Type: migrator.DB_Text, Nullable: false}, + {Name: "active", Type: migrator.DB_Bool, Nullable: false}, + {Name: "version", Type: migrator.DB_BigInt, Nullable: false}, + + // Spec + {Name: "description", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Chosen arbitrarily, but should be enough. + {Name: "keeper", Type: migrator.DB_NVarchar, Length: 253, Nullable: true}, // Keeper name, if not set, use default keeper. + {Name: "decrypters", Type: migrator.DB_Text, Nullable: true}, + {Name: "ref", Type: migrator.DB_NVarchar, Length: 1024, Nullable: true}, // Reference to third-party storage secret path.Chosen arbitrarily, but should be enough. + }, + Indices: []*migrator.Index{ + {Cols: []string{"namespace", "name", "version", "active"}, Type: migrator.UniqueIndex}, + {Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}, + }, + } + tables = append(tables, secureValueTable) + + tables = append(tables, migrator.Table{ + Name: TableNameKeeper, + Columns: []*migrator.Column{ + // Kubernetes Metadata + {Name: "guid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "annotations", Type: migrator.DB_Text, Nullable: true}, + {Name: "labels", Type: migrator.DB_Text, Nullable: true}, + {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "created_by", Type: migrator.DB_Text, Nullable: false}, + {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, // Used as RV (ResourceVersion) + {Name: "updated_by", Type: migrator.DB_Text, Nullable: false}, + + // Spec + {Name: "description", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Chosen arbitrarily, but should be enough. + {Name: "type", Type: migrator.DB_Text, Nullable: false}, + // Each keeper has a different payload so we store the whole thing as a blob. + {Name: "payload", Type: migrator.DB_Text, Nullable: true}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"namespace", "name"}, Type: migrator.UniqueIndex}, + }, + }) + + dataKeyTable := migrator.Table{ + Name: TableNameDataKey, + Columns: []*migrator.Column{ + {Name: "uid", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true}, // Arbitrarily chosen. + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "label", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: false}, // Arbitrarily chosen. + {Name: "active", Type: migrator.DB_Bool, Nullable: false}, + {Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false}, // Arbitrarily chosen. + {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, + {Name: "created", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, + }, + Indices: []*migrator.Index{}, + } + tables = append(tables, dataKeyTable) + + encryptedValueTable := migrator.Table{ + Name: TableNameEncryptedValue, + Columns: []*migrator.Column{ + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, + {Name: "version", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, + {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex}, + }, + } + tables = append(tables, encryptedValueTable) + + // Initialize all tables + for t := range tables { + mg.AddMigration("drop table "+tables[t].Name, migrator.NewDropTableMigration(tables[t].Name)) + mg.AddMigration("create table "+tables[t].Name, migrator.NewAddTableMigration(tables[t])) + for i := range tables[t].Indices { + mg.AddMigration(fmt.Sprintf("create table %s, index: %d", tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i])) + } + } + + mg.AddMigration("create index for list on "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"namespace", "active", "updated"}, + Type: migrator.IndexType, + })) + + mg.AddMigration("create index for list and read current on "+TableNameDataKey, migrator.NewAddIndexMigration(dataKeyTable, &migrator.Index{ + Cols: []string{"namespace", "label", "active"}, + Type: migrator.IndexType, + })) + + // Owner Reference columns + mg.AddMigration("add owner_reference_api_group column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_api_group", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_api_version column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_api_version", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_kind column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_kind", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add owner_reference_name column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "owner_reference_name", + Type: migrator.DB_NVarchar, + Length: 253, // Limit enforced by K8s. + Nullable: true, + })) + + mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "lease_token", + Type: migrator.DB_NVarchar, + Length: 36, + Nullable: true, + })) + mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"lease_token"}, + })) + mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{ + Name: "lease_created", + Type: migrator.DB_BigInt, + Nullable: false, + Default: "0", + })) + mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{ + Cols: []string{"lease_created"}, + })) + + encryptedValueTableUniqueKey := migrator.Index{Cols: []string{"namespace", "name", "version"}, Type: migrator.UniqueIndex} + updatedEncryptedValueTable := migrator.Table{ + Name: TableNameEncryptedValue, + Columns: []*migrator.Column{ + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, // Limit enforced by K8s. + {Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false, IsPrimaryKey: true}, + {Name: "version", Type: migrator.DB_BigInt, Nullable: false, IsPrimaryKey: true}, + {Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false}, + {Name: "created", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "updated", Type: migrator.DB_BigInt, Nullable: false}, + // {Name: "data_key_id", Type: migrator.DB_NVarchar, Length: 100, Nullable: false, Default: "''"}, // TODO: Not present until Grafana 12.3. + }, + PrimaryKeys: []string{"namespace", "name", "version"}, + // TODO: Not present until Grafana 12.3 + //Indices: []*migrator.Index{ + // {Cols: []string{"data_key_id"}}, + //}, + } + migrator.ConvertUniqueKeyToPrimaryKey(mg, encryptedValueTableUniqueKey, updatedEncryptedValueTable) + +} diff --git a/pkg/storage/unified/sql/db/migrations/resource_mig.go b/pkg/storage/unified/sql/db/migrations/resource_mig.go index 561cfdf9c70..5cc35234dbd 100644 --- a/pkg/storage/unified/sql/db/migrations/resource_mig.go +++ b/pkg/storage/unified/sql/db/migrations/resource_mig.go @@ -151,5 +151,17 @@ func initResourceTables(mg *migrator.Migrator) string { mg.AddMigration("Migrate DeletionMarkers to real Resource objects", &deletionMarkerMigrator{}) + oldResourceVersionUniqueKey := migrator.Index{Cols: []string{"group", "resource"}, Type: migrator.UniqueIndex} + updatedResourceVersionTable := migrator.Table{ + Name: "resource_version", + Columns: []*migrator.Column{ + {Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true}, + {Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false, IsPrimaryKey: true}, + {Name: "resource_version", Type: migrator.DB_BigInt, Nullable: false}, + }, + PrimaryKeys: []string{"group", "resource"}, + } + + migrator.ConvertUniqueKeyToPrimaryKey(mg, oldResourceVersionUniqueKey, updatedResourceVersionTable) return marker }