SecretsManager: Consolidation service and ability to run via cli (#108774)

* list all encrypted values and count

* separate interfaces

* add time filter to global queries

* initial secrets consolidation

* Revert defaults

* More verbose description of the operation

* Add consolidation tests and tracing

* Fix lint

* Revert debug log

pkg/server/runner.go

@@ -8,32 +8,36 @@ import (
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
+
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 )
 
 type Runner struct {
-	Cfg               *setting.Cfg
-	SQLStore          db.DB
-	SettingsProvider  setting.Provider
-	Features          featuremgmt.FeatureToggles
-	EncryptionService encryption.Internal
-	SecretsService    *manager.SecretsService
-	SecretsMigrator   secrets.Migrator
-	UserService       user.Service
+	Cfg                         *setting.Cfg
+	SQLStore                    db.DB
+	SettingsProvider            setting.Provider
+	Features                    featuremgmt.FeatureToggles
+	EncryptionService           encryption.Internal
+	SecretsService              *manager.SecretsService
+	SecretsMigrator             secrets.Migrator
+	UserService                 user.Service
+	SecretsConsolidationService contracts.ConsolidationService
 }
 
 func NewRunner(cfg *setting.Cfg, sqlStore db.DB, settingsProvider setting.Provider,
 	encryptionService encryption.Internal, features featuremgmt.FeatureToggles,
 	secretsService *manager.SecretsService, secretsMigrator secrets.Migrator,
-	userService user.Service,
+	userService user.Service, secretsConsolidationService contracts.ConsolidationService,
 ) Runner {
 	return Runner{
-		Cfg:               cfg,
-		SQLStore:          sqlStore,
-		SettingsProvider:  settingsProvider,
-		EncryptionService: encryptionService,
-		SecretsService:    secretsService,
-		SecretsMigrator:   secretsMigrator,
-		Features:          features,
-		UserService:       userService,
+		Cfg:                         cfg,
+		SQLStore:                    sqlStore,
+		SettingsProvider:            settingsProvider,
+		EncryptionService:           encryptionService,
+		SecretsService:              secretsService,
+		SecretsMigrator:             secretsMigrator,
+		Features:                    features,
+		UserService:                 userService,
+		SecretsConsolidationService: secretsConsolidationService,
 	}
 }

pkg/server/wire.go

@@ -429,7 +429,9 @@ var wireBasicSet = wire.NewSet(
 	secretdecrypt.ProvideDecryptAuthorizer,
 	secretdecrypt.ProvideDecryptService,
 	secretencryption.ProvideDataKeyStorage,
+	secretencryption.ProvideGlobalDataKeyStorage,
 	secretencryption.ProvideEncryptedValueStorage,
+	secretencryption.ProvideGlobalEncryptedValueStorage,
 	secretsecurevalueservice.ProvideSecureValueService,
 	secretvalidator.ProvideKeeperValidator,
 	secretvalidator.ProvideSecureValueValidator,

pkg/server/wireexts_oss.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	gsmKMSProviders "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper"
+	secretService "github.com/grafana/grafana/pkg/registry/apis/secret/service"
 	"github.com/grafana/grafana/pkg/registry/backgroundsvcs"
 	"github.com/grafana/grafana/pkg/registry/usagestatssvcs"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
@@ -108,6 +109,7 @@ var wireExtsBasicSet = wire.NewSet(
 	wire.Bind(new(kmsproviders.Service), new(osskmsproviders.Service)),
 	secretkeeper.ProvideService,
 	wire.Bind(new(contracts.KeeperService), new(*secretkeeper.OSSKeeperService)),
+	secretService.ProvideConsolidationService,
 	ldap.ProvideGroupsService,
 	wire.Bind(new(ldap.Groups), new(*ldap.OSSGroups)),
 	guardian.ProvideGuardian,