public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Encryption: Add support for multiple data keys per day (#47765)

Browse Source 
* Add database migrations

* Use short uids as data key ids

* Add support for manual data key rotation

* Fix duplicated mutex unlocks

* Fix migration

* Manage current data keys per name

* Adjust key re-encryption and test

* Modify rename column migration for MySQL compatibility

* Refactor secrets manager and data keys cache

* Multiple o11y adjustments

* Fix stats query

* Apply suggestions from code review

Co-authored-by: Tania <yalyna.ts@gmail.com>

* Fix linter

* Docs: Rotate data encryption keys API endpoint

Co-authored-by: Tania <yalyna.ts@gmail.com>
This commit is contained in:
Joan López de la Franca Beltran
2022-05-23 13:13:55 +02:00
committed by GitHub
co-authored by Tania
parent ae8c11bfa4
commit e43879e55d
21 changed files with 498 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/secrets/manager/cache.go
+52 -22
View File
@@ -13,6 +13,8 @@ var (
)
type dataKeyCacheEntry struct {
id string
name string
dataKey []byte
expiration time.Time
}
@@ -22,58 +24,86 @@ func (e dataKeyCacheEntry) expired() bool {
}
type dataKeyCache struct {
sync.RWMutex
entries map[string]dataKeyCacheEntry
mtx sync.RWMutex
byId map[string]*dataKeyCacheEntry
byName map[string]*dataKeyCacheEntry
cacheTTL time.Duration
}
func newDataKeyCache(ttl time.Duration) *dataKeyCache {
return &dataKeyCache{
entries: make(map[string]dataKeyCacheEntry),
byId: make(map[string]*dataKeyCacheEntry),
byName: make(map[string]*dataKeyCacheEntry),
cacheTTL: ttl,
}
}
func (c *dataKeyCache) get(id string) ([]byte, bool) {
c.RLock()
defer c.RUnlock()
func (c *dataKeyCache) getById(id string) (*dataKeyCacheEntry, bool) {
c.mtx.RLock()
defer c.mtx.RUnlock()
entry, exists := c.entries[id]
entry, exists := c.byId[id]
cacheReadsCounter.With(prometheus.Labels{
"hit": strconv.FormatBool(exists),
"hit": strconv.FormatBool(exists),
"method": "byId",
}).Inc()
if !exists || entry.expired() {
return nil, false
}
return entry.dataKey, true
return entry, true
}
func (c *dataKeyCache) add(id string, dataKey []byte) {
c.Lock()
defer c.Unlock()
func (c *dataKeyCache) getByName(name string) (*dataKeyCacheEntry, bool) {
c.mtx.RLock()
defer c.mtx.RUnlock()
c.entries[id] = dataKeyCacheEntry{
dataKey: dataKey,
expiration: now().Add(c.cacheTTL),
entry, exists := c.byName[name]
cacheReadsCounter.With(prometheus.Labels{
"hit": strconv.FormatBool(exists),
"method": "byName",
}).Inc()
if !exists || entry.expired() {
return nil, false
}
return entry, true
}
func (c *dataKeyCache) add(entry *dataKeyCacheEntry) {
c.mtx.Lock()
defer c.mtx.Unlock()
entry.expiration = now().Add(c.cacheTTL)
c.byId[entry.id] = entry
c.byName[entry.name] = entry
}
func (c *dataKeyCache) removeExpired() {
c.Lock()
defer c.Unlock()
c.mtx.Lock()
defer c.mtx.Unlock()
for id, entry := range c.entries {
for id, entry := range c.byId {
if entry.expired() {
delete(c.entries, id)
delete(c.byId, id)
}
}
for name, entry := range c.byName {
if entry.expired() {
delete(c.byName, name)
}
}
}
func (c *dataKeyCache) flush() {
c.Lock()
c.entries = make(map[string]dataKeyCacheEntry)
c.Unlock()
c.mtx.Lock()
c.byId = make(map[string]*dataKeyCacheEntry)
c.byName = make(map[string]*dataKeyCacheEntry)
c.mtx.Unlock()
}
pkg/services/secrets/manager/manager.go
+179 -92
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"strconv"
"sync"
"time"
"github.com/grafana/grafana/pkg/infra/log"
@@ -17,6 +18,7 @@ import (
"github.com/grafana/grafana/pkg/services/kmsproviders"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/errgroup"
"xorm.io/xorm"
@@ -29,10 +31,13 @@ type SecretsService struct {
features featuremgmt.FeatureToggles
usageStats usagestats.Service
currentProviderID secrets.ProviderID
mtx sync.Mutex
dataKeyCache *dataKeyCache
providers map[secrets.ProviderID]secrets.Provider
dataKeyCache *dataKeyCache
log log.Logger
currentProviderID secrets.ProviderID
log log.Logger
}
func ProvideSecretsService(
@@ -62,10 +67,9 @@ func ProvideSecretsService(
logger.Warn("Changing encryption provider requires enabling envelope encryption feature")
}
logger.Debug("Envelope encryption state", "enabled", enabled, "current provider", currentProviderID)
logger.Info("Envelope encryption state", "enabled", enabled, "current provider", currentProviderID)
ttl := settings.KeyValue("security.encryption", "data_keys_cache_ttl").MustDuration(15 * time.Minute)
cache := newDataKeyCache(ttl)
s := &SecretsService{
store: store,
@@ -73,8 +77,8 @@ func ProvideSecretsService(
settings: settings,
usageStats: usageStats,
providers: providers,
dataKeyCache: newDataKeyCache(ttl),
currentProviderID: currentProviderID,
dataKeyCache: cache,
features: features,
log: logger,
}
@@ -142,29 +146,23 @@ func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byt
// If encryption featuremgmt.FlagEnvelopeEncryption toggle is on, use envelope encryption
scope := opt()
keyName := s.keyName(scope)
name := secrets.KeyName(scope, s.currentProviderID)
var dataKey []byte
dataKey, err = s.dataKey(ctx, keyName)
id, dataKey, err := s.currentDataKey(ctx, name, scope, sess)
if err != nil {
if errors.Is(err, secrets.ErrDataKeyNotFound) {
dataKey, err = s.newDataKey(ctx, keyName, scope, sess)
if err != nil {
return nil, err
}
} else {
return nil, err
}
s.log.Error("Failed to get current data key", "error", err, "name", name)
return nil, err
}
var encrypted []byte
encrypted, err = s.enc.Encrypt(ctx, payload, string(dataKey))
if err != nil {
s.log.Error("Failed to encrypt secret", "error", err)
return nil, err
}
prefix := make([]byte, b64.EncodedLen(len(keyName))+2)
b64.Encode(prefix[1:], []byte(keyName))
prefix := make([]byte, b64.EncodedLen(len(id))+2)
b64.Encode(prefix[1:], []byte(id))
prefix[0] = '#'
prefix[len(prefix)-1] = '#'
@@ -175,8 +173,121 @@ func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byt
return blob, nil
}
func (s *SecretsService) keyName(scope string) string {
return fmt.Sprintf("%s/%s@%s", now().Format("2006-01-02"), scope, s.currentProviderID)
// currentDataKey looks up for current data key in cache or database by name, and decrypts it.
// If there's no current data key in cache nor in database it generates a new random data key,
// and stores it into both the in-memory cache and database (encrypted by the encryption provider).
func (s *SecretsService) currentDataKey(ctx context.Context, name string, scope string, sess *xorm.Session) (string, []byte, error) {
// We want only one request fetching current data key at time to
// avoid the creation of multiple ones in case there's no one existing.
s.mtx.Lock()
defer s.mtx.Unlock()
// We try to fetch the data key, either from cache or database
id, dataKey, err := s.dataKeyByName(ctx, name)
if err != nil {
return "", nil, err
}
// If no existing data key was found, create a new one
if dataKey == nil {
id, dataKey, err = s.newDataKey(ctx, name, scope, sess)
if err != nil {
return "", nil, err
}
}
return id, dataKey, nil
}
// dataKeyByName looks up for data key in cache.
// Otherwise, it fetches it from database, decrypts it and caches it decrypted.
func (s *SecretsService) dataKeyByName(ctx context.Context, name string) (string, []byte, error) {
// 0. Get data key from in-memory cache.
if entry, exists := s.dataKeyCache.getByName(name); exists {
return entry.id, entry.dataKey, nil
}
// 1. Get data key from database.
dataKey, err := s.store.GetCurrentDataKey(ctx, name)
if err != nil {
if errors.Is(err, secrets.ErrDataKeyNotFound) {
return "", nil, nil
}
return "", nil, err
}
// 2.1 Find the encryption provider.
provider, exists := s.providers[kmsproviders.NormalizeProviderID(dataKey.Provider)]
if !exists {
return "", nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
}
// 2.2 Decrypt the data key fetched from the database.
decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
if err != nil {
return "", nil, err
}
// 3. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{id: dataKey.Id, name: dataKey.Name, dataKey: decrypted})
return dataKey.Id, decrypted, nil
}
// newDataKey creates a new random data key, encrypts it and stores it into the database and cache.
func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string, sess *xorm.Session) (string, []byte, error) {
// 1. Create new data key.
dataKey, err := newRandomDataKey()
if err != nil {
return "", nil, err
}
// 2.1 Find the encryption provider.
provider, exists := s.providers[s.currentProviderID]
if !exists {
return "", nil, fmt.Errorf("could not find encryption provider '%s'", s.currentProviderID)
}
// 2.2 Encrypt the data key.
encrypted, err := provider.Encrypt(ctx, dataKey)
if err != nil {
return "", nil, err
}
// 3. Store its encrypted value into the DB.
id := util.GenerateShortUID()
dbDataKey := secrets.DataKey{
Id: id,
Active: true,
Name: name,
Provider: s.currentProviderID,
EncryptedData: encrypted,
Scope: scope,
}
if sess == nil {
err = s.store.CreateDataKey(ctx, &dbDataKey)
} else {
err = s.store.CreateDataKeyWithDBSession(ctx, &dbDataKey, sess)
}
if err != nil {
return "", nil, err
}
// 4. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{id: id, name: name, dataKey: dataKey})
return id, dataKey, nil
}
func newRandomDataKey() ([]byte, error) {
rawDataKey := make([]byte, 16)
_, err := rand.Read(rawDataKey)
if err != nil {
return nil, err
}
return rawDataKey, nil
}
func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) {
@@ -192,6 +303,10 @@ func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, e
"success": strconv.FormatBool(err == nil),
"operation": OpDecrypt,
}).Inc()
if err != nil {
s.log.Error("Failed to decrypt secret", "error", err)
}
}()
if len(payload) == 0 {
@@ -208,20 +323,20 @@ func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, e
payload = payload[1:]
endOfKey := bytes.Index(payload, []byte{'#'})
if endOfKey == -1 {
err = fmt.Errorf("could not find valid key in encrypted payload")
err = fmt.Errorf("could not find valid key id in encrypted payload")
return nil, err
}
b64Key := payload[:endOfKey]
payload = payload[endOfKey+1:]
key := make([]byte, b64.DecodedLen(len(b64Key)))
_, err = b64.Decode(key, b64Key)
keyId := make([]byte, b64.DecodedLen(len(b64Key)))
_, err = b64.Decode(keyId, b64Key)
if err != nil {
return nil, err
}
dataKey, err = s.dataKey(ctx, string(key))
dataKey, err = s.dataKeyById(ctx, string(keyId))
if err != nil {
s.log.Error("Failed to lookup data key", "name", string(key), "error", err)
s.log.Error("Failed to lookup data key by id", "id", string(keyId), "error", err)
return nil, err
}
}
@@ -275,83 +390,34 @@ func (s *SecretsService) GetDecryptedValue(ctx context.Context, sjd map[string][
return fallback
}
func newRandomDataKey() ([]byte, error) {
rawDataKey := make([]byte, 16)
_, err := rand.Read(rawDataKey)
if err != nil {
return nil, err
}
return rawDataKey, nil
}
// newDataKey creates a new random DEK, caches it and returns its value
func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string, sess *xorm.Session) ([]byte, error) {
// 1. Create new DEK
dataKey, err := newRandomDataKey()
if err != nil {
return nil, err
}
provider, exists := s.providers[s.currentProviderID]
if !exists {
return nil, fmt.Errorf("could not find encryption provider '%s'", s.currentProviderID)
// dataKeyById looks up for data key in cache.
// Otherwise, it fetches it from database and returns it decrypted.
func (s *SecretsService) dataKeyById(ctx context.Context, id string) ([]byte, error) {
// 0. Get decrypted data key from in-memory cache.
if entry, exists := s.dataKeyCache.getById(id); exists {
return entry.dataKey, nil
}
// 2. Encrypt it
encrypted, err := provider.Encrypt(ctx, dataKey)
// 1. Get encrypted data key from database.
dataKey, err := s.store.GetDataKey(ctx, id)
if err != nil {
return nil, err
}
// 3. Store its encrypted value in db
dek := secrets.DataKey{
Active: true, // TODO: right now we never mark a key as deactivated
Name: name,
Provider: s.currentProviderID,
EncryptedData: encrypted,
Scope: scope,
}
if sess == nil {
err = s.store.CreateDataKey(ctx, dek)
} else {
err = s.store.CreateDataKeyWithDBSession(ctx, dek, sess)
}
if err != nil {
return nil, err
}
// 4. Cache its unencrypted value and return it
s.dataKeyCache.add(name, dataKey)
return dataKey, nil
}
// dataKey looks up DEK in cache or database, and decrypts it
func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, error) {
if dataKey, exists := s.dataKeyCache.get(name); exists {
return dataKey, nil
}
// 1. get encrypted data key from database
dataKey, err := s.store.GetDataKey(ctx, name)
if err != nil {
return nil, err
}
// 2. decrypt data key
// 2.1. Find the encryption provider.
provider, exists := s.providers[kmsproviders.NormalizeProviderID(dataKey.Provider)]
if !exists {
return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
}
// 2.2. Encrypt the data key.
decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
if err != nil {
return nil, err
}
// 3. cache data key
s.dataKeyCache.add(name, decrypted)
// 3. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{id: id, name: dataKey.Name, dataKey: decrypted})
return decrypted, nil
}
@@ -360,17 +426,38 @@ func (s *SecretsService) GetProviders() map[secrets.ProviderID]secrets.Provider
return s.providers
}
func (s *SecretsService) ReEncryptDataKeys(ctx context.Context) error {
err := s.store.ReEncryptDataKeys(ctx, s.providers, s.currentProviderID)
func (s *SecretsService) RotateDataKeys(ctx context.Context) error {
s.log.Info("Data keys rotation triggered, acquiring lock...")
s.mtx.Lock()
defer s.mtx.Unlock()
s.log.Info("Data keys rotation started")
err := s.store.DisableDataKeys(ctx)
if err != nil {
s.log.Error("Data keys rotation failed", "error", err)
return err
}
s.dataKeyCache.flush()
s.log.Info("Data keys rotation finished successfully")
return nil
}
func (s *SecretsService) ReEncryptDataKeys(ctx context.Context) error {
s.log.Info("Data keys re-encryption triggered")
err := s.store.ReEncryptDataKeys(ctx, s.providers, s.currentProviderID)
if err != nil {
s.log.Error("Data keys re-encryption failed", "error", err)
return err
}
s.dataKeyCache.flush()
s.log.Info("Data keys re-encryption finished successfully")
return nil
}
func (s *SecretsService) Run(ctx context.Context) error {
gc := time.NewTicker(
s.settings.KeyValue("security.encryption", "data_keys_cache_cleanup_interval").
@@ -390,11 +477,11 @@ func (s *SecretsService) Run(ctx context.Context) error {
for {
select {
case <-gc.C:
s.log.Debug("removing expired data encryption keys from cache...")
s.log.Debug("Removing expired data keys from cache...")
s.dataKeyCache.removeExpired()
s.log.Debug("done removing expired data encryption keys from cache")
s.log.Debug("Removing expired data keys from cache finished successfully")
case <-gCtx.Done():
s.log.Debug("grafana is shutting down; stopping...")
s.log.Debug("Grafana is shutting down; stopping...")
gc.Stop()
if err := grp.Wait(); err != nil && !errors.Is(err, context.Canceled) {
pkg/services/secrets/manager/manager_test.go
+27 -11
View File
@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/ini.v1"
@@ -97,7 +98,8 @@ func TestSecretsService_DataKeys(t *testing.T) {
store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
ctx := context.Background()
dataKey := secrets.DataKey{
dataKey := &secrets.DataKey{
Id: util.GenerateShortUID(),
Active: true,
Name: "test1",
Provider: "test",
@@ -105,7 +107,7 @@ func TestSecretsService_DataKeys(t *testing.T) {
}
t.Run("querying for a DEK that does not exist", func(t *testing.T) {
res, err := store.GetDataKey(ctx, dataKey.Name)
res, err := store.GetDataKey(ctx, dataKey.Id)
assert.ErrorIs(t, secrets.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
@@ -114,16 +116,26 @@ func TestSecretsService_DataKeys(t *testing.T) {
err := store.CreateDataKey(ctx, dataKey)
require.NoError(t, err)
res, err := store.GetDataKey(ctx, dataKey.Name)
res, err := store.GetDataKey(ctx, dataKey.Id)
require.NoError(t, err)
assert.Equal(t, dataKey.EncryptedData, res.EncryptedData)
assert.Equal(t, dataKey.Provider, res.Provider)
assert.Equal(t, dataKey.Name, res.Name)
assert.Equal(t, dataKey.Id, res.Id)
assert.True(t, dataKey.Active)
current, err := store.GetCurrentDataKey(ctx, dataKey.Name)
require.NoError(t, err)
assert.Equal(t, dataKey.EncryptedData, current.EncryptedData)
assert.Equal(t, dataKey.Provider, current.Provider)
assert.Equal(t, dataKey.Name, current.Name)
assert.Equal(t, dataKey.Id, current.Id)
assert.True(t, current.Active)
})
t.Run("creating an inactive DEK", func(t *testing.T) {
k := secrets.DataKey{
k := &secrets.DataKey{
Id: util.GenerateShortUID(),
Active: false,
Name: "test2",
Provider: "test",
@@ -138,7 +150,7 @@ func TestSecretsService_DataKeys(t *testing.T) {
assert.Nil(t, res)
})
t.Run("deleting DEK when no name provided must fail", func(t *testing.T) {
t.Run("deleting DEK when no id provided must fail", func(t *testing.T) {
beforeDelete, err := store.GetAllDataKeys(ctx)
require.NoError(t, err)
err = store.DeleteDataKey(ctx, "")
@@ -150,10 +162,10 @@ func TestSecretsService_DataKeys(t *testing.T) {
})
t.Run("deleting a DEK", func(t *testing.T) {
err := store.DeleteDataKey(ctx, dataKey.Name)
err := store.DeleteDataKey(ctx, dataKey.Id)
require.NoError(t, err)
res, err := store.GetDataKey(ctx, dataKey.Name)
res, err := store.GetDataKey(ctx, dataKey.Id)
assert.Equal(t, secrets.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
@@ -280,7 +292,8 @@ func TestSecretsService_Run(t *testing.T) {
require.NoError(t, err)
// Data encryption key cache should contain one element
require.Len(t, svc.dataKeyCache.entries, 1)
require.Len(t, svc.dataKeyCache.byId, 1)
require.Len(t, svc.dataKeyCache.byName, 1)
t.Cleanup(func() { now = time.Now })
now = func() time.Time { return time.Now().Add(10 * time.Minute) }
@@ -294,7 +307,8 @@ func TestSecretsService_Run(t *testing.T) {
// Then, once the ticker has been triggered,
// the cleanup process should have happened,
// therefore the cache should be empty.
require.Len(t, svc.dataKeyCache.entries, 0)
require.Len(t, svc.dataKeyCache.byId, 0)
require.Len(t, svc.dataKeyCache.byName, 0)
})
}
@@ -328,11 +342,13 @@ func TestSecretsService_ReEncryptDataKeys(t *testing.T) {
// Decrypt to ensure data key is cached
_, err := svc.Decrypt(ctx, ciphertext)
require.NoError(t, err)
require.NotEmpty(t, svc.dataKeyCache.entries)
require.NotEmpty(t, svc.dataKeyCache.byId)
require.NotEmpty(t, svc.dataKeyCache.byName)
err = svc.ReEncryptDataKeys(ctx)
require.NoError(t, err)
assert.Empty(t, svc.dataKeyCache.entries)
assert.Empty(t, svc.dataKeyCache.byId)
assert.Empty(t, svc.dataKeyCache.byName)
})
}
pkg/services/secrets/manager/metrics.go
+1 -1
View File
@@ -25,7 +25,7 @@ var (
Name: "encryption_cache_reads_total",
Help: "A counter for encryption cache reads",
},
[]string{"hit"},
[]string{"hit", "method"},
)
)