public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
736d035c657109dfd4e4961d5aaf06f8d9846da4
grafana/pkg/services/datasources/service/secrets_mig_test.go
T
Guilherme Caulada 2d8a91a846 Secrets: Improve unified secrets migration and implement compatibility flag (#50463) 
* Implement disableSecretsCompatibility flag

* Allow secret deletion right after migration

* Use dialect.Quote for secure_json_data on secret deletion

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Set secure_json_data to NULL instead of empty json

* Run toggles_gen_test and use generated flag variable

* Add ID to delete data source secrets command on function call

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Remove extra query to get datasource on secret deletion

* Fix linting issues with CHANGELOG.md

* Use empty json string when deleting secure json data

* Implement secret migration as a background process

* Refactor secret migration as a background service

* Refactor migration to be inside secret store

* Re-add secret deletion function removed on merge

* Try using transaction to fix db lock during tests

* Disable migration for pipeline debugging

* Try adding sleep to fix database lock

* Remove unecessary time sleep from migration

* Fix merge issue, replace models with datasources

* Try event listener approach

* Fix merge issue, replace models with datasources

* Fix linting issues with unchecked error

* Remove unecessary trainling new line

* Increase wait interval on background secret migration

* Rename secret store migration folder for consistency

* Convert background migration to blocking

* Fix number of arguments on server tests

* Check error value of secret migration provider

* Fix linting issue with method varaible

* Revert unintended change on background services

* Move secret migration service provider to wire.go

* Remove unecessary else from datasource service

* Move transaction inside loop on secret migration

* Remove unecessary GetServices function

* Remove unecessary interface after method removal

* Rename Run to Migrate on secret migration interface

* Rename secret migrations service variable on server

* Use MustBool on datasource secret migration

* Revert changes to GetDataSources

* Implement GetAllDataSources function

* Remove DeleteDataSourceSecrets function

* Move datasource secret migration to datasource service

* Remove unecessary properties from datasource secret migration

* Make DecryptLegacySecrets a private method

* Remove context canceled check on secret migrator

* Log error when fail to unmarshal datasource secret

* Add necessary fields to update command on migration

* Handle high availability on secret migration

* Use kvstore for datasource secret migration status

* Add error check for migration status set on kvstore

* Remove NewSecretMigrationService from server tests

* Use const for strings on datasource secrets migration

* Test all cases for datasources secret migrations

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>
2022-07-12 17:27:37 -03:00

341 lines
13 KiB
Go
Raw Blame History

package service
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/infra/kvstore"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
)
func SetupTestMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, secretsStore secretsStore.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
t.Helper()
cfg := &setting.Cfg{}
features := featuremgmt.WithFeatures()
if !compatibility {
features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
}
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New().WithDisabled(), acmock.NewMockedPermissionsService())
migService := ProvideDataSourceMigrationService(dsService, kvStore, features)
return migService
}
func TestMigrate(t *testing.T) {
t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration without compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration with compatibility
migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was re-added for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
})
t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
sqlStore := sqlstore.InitTestDB(t)
kvStore := kvstore.ProvideService(sqlStore)
secretsStore := secretsStore.SetupTestService(t)
migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
dataSourceName := "Test"
dataSourceOrg := int64(1)
// Add test data source
err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
OrgId: dataSourceOrg,
Name: dataSourceName,
Type: datasources.DS_MYSQL,
Access: datasources.DS_ACCESS_DIRECT,
Url: "http://test",
EncryptedSecureJsonData: map[string][]byte{
"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
},
})
assert.NoError(t, err)
// Check if the secret json data was added
query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the migration status key is empty
value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Check that the secret is not present on the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.Empty(t, value)
assert.False(t, exist)
// Run the migration with compatibility
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was maintained for compatibility
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.NotEmpty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, compatibleSecretMigrationValue, value)
assert.True(t, exist)
// Run the migration without compatibility
migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
err = migService.Migrate(context.Background())
assert.NoError(t, err)
// Check if the secure json data was deleted
query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
err = sqlStore.GetDataSource(context.Background(), query)
assert.NoError(t, err)
assert.NotNil(t, query.Result)
assert.Empty(t, query.Result.SecureJsonData)
// Check if the secret was added to the secret store
value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
assert.NoError(t, err)
assert.NotEmpty(t, value)
assert.True(t, exist)
// Check if the migration status key was set
value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
assert.NoError(t, err)
assert.Equal(t, completeSecretMigrationValue, value)
assert.True(t, exist)
})
}
Reference in New Issue View Git Blame Copy Permalink