public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove flag disableSecretsCompatibility (#103135)

Browse Source 
Remove flag disableSecretsCompatibility
This commit is contained in:
Tania
2025-04-23 16:21:13 +00:00
committed by GitHub
parent 2e52553a39
commit b8eeea0fe0
8 changed files with 259 additions and 851 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/secrets/kvstore/migrations/datasource_mig.go
+6 -22
View File
@@ -13,10 +13,8 @@ import (
const (
// Not set means migration has not happened
secretMigrationStatusKey = "secretMigrationStatus"
// Migration happened with disableSecretCompatibility set to false
// Migration happened and secrets are stored in both locations
compatibleSecretMigrationValue = "compatible"
// Migration happened with disableSecretCompatibility set to true
completeSecretMigrationValue = "complete"
)
type DataSourceSecretMigrationService struct {
@@ -43,17 +41,10 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
return err
}
logger.Debug(fmt.Sprint("secret migration status is ", migrationStatus))
// If this flag is true, delete secrets from the legacy secrets store as they are migrated
disableSecretsCompatibility := s.features.IsEnabled(ctx, featuremgmt.FlagDisableSecretsCompatibility)
// If migration hasn't happened, migrate to unified secrets and keep copy in legacy
// If a complete migration happened and now backwards compatibility is enabled, copy secrets back to legacy
needCompatibility := migrationStatus != compatibleSecretMigrationValue && !disableSecretsCompatibility
// If migration hasn't happened, migrate to unified secrets and delete from legacy
// If a compatible migration happened and now compatibility is disabled, delete secrets from legacy
needMigration := migrationStatus != completeSecretMigrationValue && disableSecretsCompatibility
if needCompatibility || needMigration {
logger.Debug("performing secret migration", "needs migration", needMigration, "needs compatibility", needCompatibility)
// Only migrate if it hasn't happened yet
if migrationStatus != compatibleSecretMigrationValue {
logger.Debug("performing secret migration")
query := &datasources.GetAllDataSourcesQuery{}
dsList, err := s.dataSourcesService.GetAllDataSources(ctx, query)
if err != nil {
@@ -67,7 +58,6 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
}
// Secrets are set by the update data source function if the SecureJsonData is set in the command
// Secrets are deleted by the update data source function if the disableSecretsCompatibility flag is enabled
_, err = s.dataSourcesService.UpdateDataSource(ctx, &datasources.UpdateDataSourceCommand{
ID: ds.ID,
OrgID: ds.OrgID,
@@ -89,17 +79,11 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
}
}
var newMigStatus string
if disableSecretsCompatibility {
newMigStatus = completeSecretMigrationValue
} else {
newMigStatus = compatibleSecretMigrationValue
}
err = s.kvStore.Set(ctx, secretMigrationStatusKey, newMigStatus)
err = s.kvStore.Set(ctx, secretMigrationStatusKey, compatibleSecretMigrationValue)
if err != nil {
return err
}
logger.Debug(fmt.Sprint("set secret migration status to ", newMigStatus))
logger.Debug(fmt.Sprint("set secret migration status to ", compatibleSecretMigrationValue))
}
return nil
pkg/services/secrets/kvstore/migrations/datasource_mig_test.go
+2 -252
View File
@@ -30,13 +30,10 @@ func TestMain(m *testing.M) {
testsuite.Run(m)
}
func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvStore kvstore.KVStore, secretsStore secretskvs.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvStore kvstore.KVStore, secretsStore secretskvs.SecretsKVStore) *DataSourceSecretMigrationService {
t.Helper()
cfg := &setting.Cfg{}
features := featuremgmt.WithFeatures()
if !compatibility {
features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
}
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
quotaService := quotatest.New(false, nil)
dsService, err := dsservice.ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New(),
@@ -48,76 +45,12 @@ func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvS
}
func TestMigrate(t *testing.T) {
t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
@@ -177,187 +110,4 @@ func TestMigrate(t *testing.T) {
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration without compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration with compatibility
migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was re-added for compatibility
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration with compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration without compatibility
migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
}