From 773baf47e18394a0b904a84043d548b391dd0a7b Mon Sep 17 00:00:00 2001 From: Michael Mandrus Date: Thu, 13 Nov 2025 15:33:25 -0500 Subject: [PATCH] pass dek cache into encryption manager --- pkg/extensions/enterprise_imports.go | 1 - .../secret/encryption/manager/dek_cache.go | 104 ++++++++++++++++++ .../apis/secret/encryption/manager/manager.go | 35 ++++++ .../secret/encryption/manager/manager_test.go | 18 +++ .../secret/encryption/manager/test_helpers.go | 25 +++++ .../apis/secret/encryption/secrets.go | 22 ++++ .../secret/secretkeeper/secretkeeper_test.go | 2 +- .../apis/secret/testutils/testutils.go | 2 + pkg/server/wire_gen.go | 9 +- pkg/server/wireexts_oss.go | 2 + pkg/setting/setting_secrets_manager.go | 12 ++ 11 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 pkg/registry/apis/secret/encryption/manager/dek_cache.go diff --git a/pkg/extensions/enterprise_imports.go b/pkg/extensions/enterprise_imports.go index 113c2f8e4bb..9c052688d8f 100644 --- a/pkg/extensions/enterprise_imports.go +++ b/pkg/extensions/enterprise_imports.go @@ -58,5 +58,4 @@ import ( _ "github.com/grafana/grafana/apps/alerting/alertenrichment/pkg/apis/alertenrichment/v1beta1" _ "github.com/grafana/grafana/apps/scope/pkg/apis/scope/v0alpha1" - _ "github.com/testcontainers/testcontainers-go" ) diff --git a/pkg/registry/apis/secret/encryption/manager/dek_cache.go b/pkg/registry/apis/secret/encryption/manager/dek_cache.go new file mode 100644 index 00000000000..9bc39794eb5 --- /dev/null +++ b/pkg/registry/apis/secret/encryption/manager/dek_cache.go @@ -0,0 +1,104 @@ +package manager + +import ( + "strconv" + "sync" + "time" + + "github.com/grafana/grafana/pkg/registry/apis/secret/encryption" + "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" +) + +type ossDataKeyCache struct { + mtx sync.RWMutex + byId map[string]*encryption.DataKeyCacheEntry + byLabel map[string]*encryption.DataKeyCacheEntry + cacheTTL time.Duration +} + +func ProvideOSSDataKeyCache(cfg *setting.Cfg) encryption.DataKeyCache { + return &ossDataKeyCache{ + byId: make(map[string]*encryption.DataKeyCacheEntry), + byLabel: make(map[string]*encryption.DataKeyCacheEntry), + cacheTTL: cfg.SecretsManagement.DataKeysCacheTTL, + } +} + +func (c *ossDataKeyCache) GetById(id string) (*encryption.DataKeyCacheEntry, bool) { + c.mtx.RLock() + defer c.mtx.RUnlock() + + entry, exists := c.byId[id] + + cacheReadsCounter.With(prometheus.Labels{ + "hit": strconv.FormatBool(exists), + "method": "byId", + }).Inc() + + if !exists || entry.IsExpired() { + return nil, false + } + + return entry, true +} + +func (c *ossDataKeyCache) GetByLabel(label string) (*encryption.DataKeyCacheEntry, bool) { + c.mtx.RLock() + defer c.mtx.RUnlock() + + entry, exists := c.byLabel[label] + + cacheReadsCounter.With(prometheus.Labels{ + "hit": strconv.FormatBool(exists), + "method": "byLabel", + }).Inc() + + if !exists || entry.IsExpired() { + return nil, false + } + + return entry, true +} + +func (c *ossDataKeyCache) AddById(entry *encryption.DataKeyCacheEntry) { + c.mtx.Lock() + defer c.mtx.Unlock() + + entry.Expiration = time.Now().Add(c.cacheTTL) + + c.byId[entry.Id] = entry +} + +func (c *ossDataKeyCache) AddByLabel(entry *encryption.DataKeyCacheEntry) { + c.mtx.Lock() + defer c.mtx.Unlock() + + entry.Expiration = time.Now().Add(c.cacheTTL) + + c.byLabel[entry.Label] = entry +} + +func (c *ossDataKeyCache) RemoveExpired() { + c.mtx.Lock() + defer c.mtx.Unlock() + + for id, entry := range c.byId { + if entry.IsExpired() { + delete(c.byId, id) + } + } + + for label, entry := range c.byLabel { + if entry.IsExpired() { + delete(c.byLabel, label) + } + } +} + +func (c *ossDataKeyCache) Flush() { + c.mtx.Lock() + c.byId = make(map[string]*encryption.DataKeyCacheEntry) + c.byLabel = make(map[string]*encryption.DataKeyCacheEntry) + c.mtx.Unlock() +} diff --git a/pkg/registry/apis/secret/encryption/manager/manager.go b/pkg/registry/apis/secret/encryption/manager/manager.go index 024c8aca05a..da456e1bac8 100644 --- a/pkg/registry/apis/secret/encryption/manager/manager.go +++ b/pkg/registry/apis/secret/encryption/manager/manager.go @@ -7,11 +7,13 @@ import ( "fmt" "strconv" "sync" + "time" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/usagestats" @@ -19,6 +21,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/secret/encryption" "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher" "github.com/grafana/grafana/pkg/registry/apis/secret/xkube" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -26,6 +29,9 @@ type EncryptionManager struct { tracer trace.Tracer store contracts.DataKeyStorage usageStats usagestats.Service + cfg *setting.Cfg + + dataKeyCache encryption.DataKeyCache mtx sync.Mutex @@ -44,6 +50,8 @@ func ProvideEncryptionManager( usageStats usagestats.Service, enc cipher.Cipher, providerConfig encryption.ProviderConfig, + dataKeyCache encryption.DataKeyCache, + cfg *setting.Cfg, ) (contracts.EncryptionManager, error) { currentProviderID := providerConfig.CurrentProvider if _, ok := providerConfig.AvailableProviders[currentProviderID]; !ok { @@ -57,6 +65,7 @@ func ProvideEncryptionManager( cipher: enc, log: log.New("encryption"), providerConfig: providerConfig, + cfg: cfg, } s.registerUsageMetrics() @@ -327,3 +336,29 @@ func (s *EncryptionManager) dataKeyById(ctx context.Context, namespace, id strin func (s *EncryptionManager) GetProviders() encryption.ProviderConfig { return s.providerConfig } + +func (s *EncryptionManager) Run(ctx context.Context) error { + gc := time.NewTicker( + s.cfg.SecretsManagement.DataKeysCacheCleanupInterval, + ) + + grp, gCtx := errgroup.WithContext(ctx) + + for { + select { + case <-gc.C: + s.log.Debug("Removing expired data keys from cache...") + s.dataKeyCache.RemoveExpired() + s.log.Debug("Removing expired data keys from cache finished successfully") + case <-gCtx.Done(): + s.log.Debug("Grafana is shutting down; stopping...") + gc.Stop() + + if err := grp.Wait(); err != nil && !errors.Is(err, context.Canceled) { + return err + } + + return nil + } + } +} diff --git a/pkg/registry/apis/secret/encryption/manager/manager_test.go b/pkg/registry/apis/secret/encryption/manager/manager_test.go index dc63fb461c9..409253f2141 100644 --- a/pkg/registry/apis/secret/encryption/manager/manager_test.go +++ b/pkg/registry/apis/secret/encryption/manager/manager_test.go @@ -201,6 +201,8 @@ func TestEncryptionService_UseCurrentProvider(t *testing.T) { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfg, ) require.NoError(t, err) @@ -226,6 +228,8 @@ func TestEncryptionService_UseCurrentProvider(t *testing.T) { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfg, ) require.NoError(t, err) @@ -275,6 +279,8 @@ func TestEncryptionService_SecretKeyVersionUpgrade(t *testing.T) { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfgV1, ) require.NoError(t, err) @@ -313,6 +319,8 @@ func TestEncryptionService_SecretKeyVersionUpgrade(t *testing.T) { usageStats, enc, ossProvidersV2, + &NoopDataKeyCache{}, + cfgV2, ) require.NoError(t, err) @@ -368,6 +376,8 @@ func TestEncryptionService_SecretKeyVersionUpgrade(t *testing.T) { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfgV1, ) require.NoError(t, err) @@ -392,6 +402,8 @@ func TestEncryptionService_SecretKeyVersionUpgrade(t *testing.T) { usageStats, enc, ossProvidersV2, + &NoopDataKeyCache{}, + cfgV2, ) require.NoError(t, err) @@ -573,6 +585,8 @@ func TestIntegration_SecretsService(t *testing.T) { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfg, ) require.NoError(t, err) @@ -610,6 +624,8 @@ func TestEncryptionService_ThirdPartyProviders(t *testing.T) { enc, err := service.ProvideAESGCMCipherService(tracer, usageStats) require.NoError(t, err) + cfg := &setting.Cfg{} + svc, err := ProvideEncryptionManager( tracer, nil, @@ -621,6 +637,8 @@ func TestEncryptionService_ThirdPartyProviders(t *testing.T) { encryption.ProviderID("fakeProvider.v1"): &fakeProvider{}, }, }, + &NoopDataKeyCache{}, + cfg, ) require.NoError(t, err) diff --git a/pkg/registry/apis/secret/encryption/manager/test_helpers.go b/pkg/registry/apis/secret/encryption/manager/test_helpers.go index d8dfe9a0650..662ca88ca9c 100644 --- a/pkg/registry/apis/secret/encryption/manager/test_helpers.go +++ b/pkg/registry/apis/secret/encryption/manager/test_helpers.go @@ -7,6 +7,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/grafana/grafana/pkg/infra/usagestats" + "github.com/grafana/grafana/pkg/registry/apis/secret/encryption" "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service" osskmsproviders "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -47,8 +48,32 @@ func setupTestService(tb testing.TB) *EncryptionManager { usageStats, enc, ossProviders, + &NoopDataKeyCache{}, + cfg, ) require.NoError(tb, err) return encMgr.(*EncryptionManager) } + +type NoopDataKeyCache struct { +} + +func (c *NoopDataKeyCache) GetById(id string) (*encryption.DataKeyCacheEntry, bool) { + return nil, false +} + +func (c *NoopDataKeyCache) GetByLabel(label string) (*encryption.DataKeyCacheEntry, bool) { + return nil, false +} + +func (c *NoopDataKeyCache) AddById(entry *encryption.DataKeyCacheEntry) { +} + +func (c *NoopDataKeyCache) AddByLabel(entry *encryption.DataKeyCacheEntry) { +} + +func (c *NoopDataKeyCache) RemoveExpired() { +} + +func (c *NoopDataKeyCache) Flush() {} diff --git a/pkg/registry/apis/secret/encryption/secrets.go b/pkg/registry/apis/secret/encryption/secrets.go index 426cbb79543..20fcbe0c56a 100644 --- a/pkg/registry/apis/secret/encryption/secrets.go +++ b/pkg/registry/apis/secret/encryption/secrets.go @@ -40,3 +40,25 @@ func (id ProviderID) Kind() (string, error) { func KeyLabel(providerID ProviderID) string { return fmt.Sprintf("%s@%s", time.Now().Format("2006-01-02"), providerID) } + +type DataKeyCache interface { + GetById(id string) (*DataKeyCacheEntry, bool) + GetByLabel(label string) (*DataKeyCacheEntry, bool) + AddById(entry *DataKeyCacheEntry) + AddByLabel(entry *DataKeyCacheEntry) + RemoveExpired() + Flush() +} + +type DataKeyCacheEntry struct { + Namespace string + Id string + Label string + DataKey []byte + Active bool + Expiration time.Time +} + +func (e DataKeyCacheEntry) IsExpired() bool { + return e.Expiration.Before(time.Now()) +} diff --git a/pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go b/pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go index f60dcc91188..eb86199047c 100644 --- a/pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go +++ b/pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go @@ -62,7 +62,7 @@ func setupTestService(t *testing.T, cfg *setting.Cfg) (*OSSKeeperService, error) ossProviders, err := osskmsproviders.ProvideOSSKMSProviders(cfg, enc) require.NoError(t, err) - encryptionManager, err := manager.ProvideEncryptionManager(tracer, dataKeyStore, usageStats, enc, ossProviders) + encryptionManager, err := manager.ProvideEncryptionManager(tracer, dataKeyStore, usageStats, enc, ossProviders, &manager.NoopDataKeyCache{}, cfg) require.NoError(t, err) // Initialize the keeper service diff --git a/pkg/registry/apis/secret/testutils/testutils.go b/pkg/registry/apis/secret/testutils/testutils.go index 6a5f1e40d30..2e1a138f930 100644 --- a/pkg/registry/apis/secret/testutils/testutils.go +++ b/pkg/registry/apis/secret/testutils/testutils.go @@ -120,6 +120,8 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut { usageStats, enc, ossProviders, + &manager.NoopDataKeyCache{}, + cfg, ) require.NoError(t, err) diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index b10e7b81e3d..8c7c6b30b96 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -482,7 +482,8 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } - encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig) + dataKeyCache := manager2.ProvideOSSDataKeyCache(cfg) + encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig, dataKeyCache, cfg) if err != nil { return nil, err } @@ -1122,7 +1123,8 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig) + dataKeyCache := manager2.ProvideOSSDataKeyCache(cfg) + encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig, dataKeyCache, cfg) if err != nil { return nil, err } @@ -1658,7 +1660,8 @@ func InitializeForCLI(ctx context.Context, cfg *setting.Cfg) (Runner, error) { if err != nil { return Runner{}, err } - encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig) + dataKeyCache := manager2.ProvideOSSDataKeyCache(cfg) + encryptionManager, err := manager2.ProvideEncryptionManager(tracer, dataKeyStorage, usageStats, cipher, providerConfig, dataKeyCache, cfg) if err != nil { return Runner{}, err } diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index b2b134ee4a9..81eb9da01c4 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/secret" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" gsmKMSProviders "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders" + gsmEncryptionManager "github.com/grafana/grafana/pkg/registry/apis/secret/encryption/manager" "github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper" secretService "github.com/grafana/grafana/pkg/registry/apis/secret/service" "github.com/grafana/grafana/pkg/registry/apps/advisor" @@ -148,6 +149,7 @@ var wireExtsBasicSet = wire.NewSet( aggregatorrunner.ProvideNoopAggregatorConfigurator, apisregistry.WireSetExts, gsmKMSProviders.ProvideOSSKMSProviders, + gsmEncryptionManager.ProvideOSSDataKeyCache, secret.ProvideSecureValueClient, provisioningExtras, configProviderExtras, diff --git a/pkg/setting/setting_secrets_manager.go b/pkg/setting/setting_secrets_manager.go index d05e6f40f85..22eb3fe27a1 100644 --- a/pkg/setting/setting_secrets_manager.go +++ b/pkg/setting/setting_secrets_manager.go @@ -11,8 +11,16 @@ const ( ) type SecretsManagerSettings struct { + // Which encryption provider to use to encrypt any new secrets CurrentEncryptionProvider string + // The time to live for decrypted data keys in memory + DataKeysCacheTTL time.Duration + // The interval to remove expired data keys from the cache + DataKeysCacheCleanupInterval time.Duration + // Whether to use a Redis cache for data keys instead of the in-memory cache + DataKeysCacheUseRedis bool + // ConfiguredKMSProviders is a map of KMS providers found in the config file. The keys are in the format of ., and the values are a map of the properties in that section // In OSS, the provider type can only be "secret_key". In Enterprise, it can additionally be one of: "aws_kms", "azure_keyvault", "google_kms", "hashicorp_vault" ConfiguredKMSProviders map[string]map[string]string @@ -66,6 +74,10 @@ func (cfg *Cfg) readSecretsManagerSettings() { cfg.SecretsManagement.RunSecretsDBMigrations = secretsMgmt.Key("run_secrets_db_migrations").MustBool(true) cfg.SecretsManagement.RunDataKeyMigration = secretsMgmt.Key("run_data_key_migration").MustBool(true) + cfg.SecretsManagement.DataKeysCacheUseRedis = secretsMgmt.Key("data_keys_cache_use_redis").MustBool(false) + cfg.SecretsManagement.DataKeysCacheTTL = secretsMgmt.Key("data_keys_cache_ttl").MustDuration(15 * time.Minute) + cfg.SecretsManagement.DataKeysCacheCleanupInterval = secretsMgmt.Key("data_keys_cache_cleanup_interval").MustDuration(1 * time.Minute) + // Extract available KMS providers from configuration sections providers := make(map[string]map[string]string) for _, section := range cfg.Raw.Sections() {