public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove flag disableSecretsCompatibility (#103135)

Browse Source 
Remove flag disableSecretsCompatibility
This commit is contained in:
Tania
2025-04-23 16:21:13 +00:00
committed by GitHub
parent 2e52553a39
commit b8eeea0fe0
8 changed files with 259 additions and 851 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/types/featureToggles.gen.ts
-4
View File
@@ -59,10 +59,6 @@ export interface FeatureToggles {
*/
canvasPanelNesting?: boolean;
/**
* Disable duplicated secret storage in legacy tables
*/
disableSecretsCompatibility?: boolean;
/**
* Logs the path for requests that are instrumented as unknown
*/
logRequestsInstrumentedAsUnknown?: boolean;
pkg/services/datasources/service/datasource.go
+6 -10
View File
@@ -259,11 +259,9 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *datasources.AddDataSou
var err error
cmd.EncryptedSecureJsonData = make(map[string][]byte)
if !s.features.IsEnabled(ctx, featuremgmt.FlagDisableSecretsCompatibility) {
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
if err != nil {
return err
}
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
if err != nil {
return err
}
cmd.UpdateSecretFn = func() error {
@@ -948,11 +946,9 @@ func (s *Service) fillWithSecureJSONData(ctx context.Context, cmd *datasources.U
}
cmd.EncryptedSecureJsonData = make(map[string][]byte)
if !s.features.IsEnabled(ctx, featuremgmt.FlagDisableSecretsCompatibility) {
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
if err != nil {
return err
}
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
if err != nil {
return err
}
return nil
pkg/services/featuremgmt/registry.go
-7
View File
@@ -86,13 +86,6 @@ var (
Owner: grafanaDatavizSquad,
HideFromAdminPage: true,
},
{
Name: "disableSecretsCompatibility",
Description: "Disable duplicated secret storage in legacy tables",
Stage: FeatureStageExperimental,
RequiresRestart: true,
Owner: hostedGrafanaTeam,
},
{
Name: "logRequestsInstrumentedAsUnknown",
Description: "Logs the path for requests that are instrumented as unknown",
pkg/services/featuremgmt/toggles_gen.csv
-1
View File
@@ -8,7 +8,6 @@ featureHighlights,GA,@grafana/grafana-as-code,false,false,false
storage,experimental,@grafana/search-and-storage,false,false,false
correlations,GA,@grafana/dataviz-squad,false,false,false
canvasPanelNesting,experimental,@grafana/dataviz-squad,false,false,true
disableSecretsCompatibility,experimental,@grafana/hosted-grafana-team,false,true,false
logRequestsInstrumentedAsUnknown,experimental,@grafana/hosted-grafana-team,false,false,false
grpcServer,preview,@grafana/search-and-storage,false,false,false
cloudWatchCrossAccountQuerying,GA,@grafana/aws-datasources,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
8 storage experimental @grafana/search-and-storage false false false
9 correlations GA @grafana/dataviz-squad false false false
10 canvasPanelNesting experimental @grafana/dataviz-squad false false true
disableSecretsCompatibility experimental @grafana/hosted-grafana-team false true false
11 logRequestsInstrumentedAsUnknown experimental @grafana/hosted-grafana-team false false false
12 grpcServer preview @grafana/search-and-storage false false false
13 cloudWatchCrossAccountQuerying GA @grafana/aws-datasources false false false
pkg/services/featuremgmt/toggles_gen.go
-4
View File
@@ -43,10 +43,6 @@ const (
// Allow elements nesting
FlagCanvasPanelNesting = "canvasPanelNesting"
// FlagDisableSecretsCompatibility
// Disable duplicated secret storage in legacy tables
FlagDisableSecretsCompatibility = "disableSecretsCompatibility"
// FlagLogRequestsInstrumentedAsUnknown
// Logs the path for requests that are instrumented as unknown
FlagLogRequestsInstrumentedAsUnknown = "logRequestsInstrumentedAsUnknown"
pkg/services/featuremgmt/toggles_gen.json
+245 -551
View File
File diff suppressed because it is too large Load Diff
pkg/services/secrets/kvstore/migrations/datasource_mig.go
+6 -22
View File
@@ -13,10 +13,8 @@ import (
const (
// Not set means migration has not happened
secretMigrationStatusKey = "secretMigrationStatus"
// Migration happened with disableSecretCompatibility set to false
// Migration happened and secrets are stored in both locations
compatibleSecretMigrationValue = "compatible"
// Migration happened with disableSecretCompatibility set to true
completeSecretMigrationValue = "complete"
)
type DataSourceSecretMigrationService struct {
@@ -43,17 +41,10 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
return err
}
logger.Debug(fmt.Sprint("secret migration status is ", migrationStatus))
// If this flag is true, delete secrets from the legacy secrets store as they are migrated
disableSecretsCompatibility := s.features.IsEnabled(ctx, featuremgmt.FlagDisableSecretsCompatibility)
// If migration hasn't happened, migrate to unified secrets and keep copy in legacy
// If a complete migration happened and now backwards compatibility is enabled, copy secrets back to legacy
needCompatibility := migrationStatus != compatibleSecretMigrationValue && !disableSecretsCompatibility
// If migration hasn't happened, migrate to unified secrets and delete from legacy
// If a compatible migration happened and now compatibility is disabled, delete secrets from legacy
needMigration := migrationStatus != completeSecretMigrationValue && disableSecretsCompatibility
if needCompatibility || needMigration {
logger.Debug("performing secret migration", "needs migration", needMigration, "needs compatibility", needCompatibility)
// Only migrate if it hasn't happened yet
if migrationStatus != compatibleSecretMigrationValue {
logger.Debug("performing secret migration")
query := &datasources.GetAllDataSourcesQuery{}
dsList, err := s.dataSourcesService.GetAllDataSources(ctx, query)
if err != nil {
@@ -67,7 +58,6 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
}
// Secrets are set by the update data source function if the SecureJsonData is set in the command
// Secrets are deleted by the update data source function if the disableSecretsCompatibility flag is enabled
_, err = s.dataSourcesService.UpdateDataSource(ctx, &datasources.UpdateDataSourceCommand{
ID: ds.ID,
OrgID: ds.OrgID,
@@ -89,17 +79,11 @@ func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
}
}
var newMigStatus string
if disableSecretsCompatibility {
newMigStatus = completeSecretMigrationValue
} else {
newMigStatus = compatibleSecretMigrationValue
}
err = s.kvStore.Set(ctx, secretMigrationStatusKey, newMigStatus)
err = s.kvStore.Set(ctx, secretMigrationStatusKey, compatibleSecretMigrationValue)
if err != nil {
return err
}
logger.Debug(fmt.Sprint("set secret migration status to ", newMigStatus))
logger.Debug(fmt.Sprint("set secret migration status to ", compatibleSecretMigrationValue))
}
return nil
pkg/services/secrets/kvstore/migrations/datasource_mig_test.go
+2 -252
View File
@@ -30,13 +30,10 @@ func TestMain(m *testing.M) {
testsuite.Run(m)
}
func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvStore kvstore.KVStore, secretsStore secretskvs.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvStore kvstore.KVStore, secretsStore secretskvs.SecretsKVStore) *DataSourceSecretMigrationService {
t.Helper()
cfg := &setting.Cfg{}
features := featuremgmt.WithFeatures()
if !compatibility {
features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
}
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
quotaService := quotatest.New(false, nil)
dsService, err := dsservice.ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New(),
@@ -48,76 +45,12 @@ func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore db.DB, kvS
}
func TestMigrate(t *testing.T) {
t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
@@ -177,187 +110,4 @@ func TestMigrate(t *testing.T) {
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration without compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration with compatibility
migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was re-added for compatibility
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
ds := dsservice.CreateStore(sqlStore, log.NewNopLogger())
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
_, err := ds.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgID: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
URL: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err := ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration with compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.NotEmpty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration without compatibility
migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgID: dataSourceOrg, Name: dataSourceName}
dataSource, err = ds.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, dataSource)
assert.Empty(t, dataSource.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
}