Encryption: Add support for multiple data keys per day (#47765)

* Add database migrations

* Use short uids as data key ids

* Add support for manual data key rotation

* Fix duplicated mutex unlocks

* Fix migration

* Manage current data keys per name

* Adjust key re-encryption and test

* Modify rename column migration for MySQL compatibility

* Refactor secrets manager and data keys cache

* Multiple o11y adjustments

* Fix stats query

* Apply suggestions from code review

Co-authored-by: Tania <yalyna.ts@gmail.com>

* Fix linter

* Docs: Rotate data encryption keys API endpoint

Co-authored-by: Tania <yalyna.ts@gmail.com>

pkg/services/secrets/manager/cache.go

@@ -13,6 +13,8 @@ var (
 )
 
 type dataKeyCacheEntry struct {
+	id         string
+	name       string
 	dataKey    []byte
 	expiration time.Time
 }
@@ -22,58 +24,86 @@ func (e dataKeyCacheEntry) expired() bool {
 }
 
 type dataKeyCache struct {
-	sync.RWMutex
-	entries  map[string]dataKeyCacheEntry
+	mtx      sync.RWMutex
+	byId     map[string]*dataKeyCacheEntry
+	byName   map[string]*dataKeyCacheEntry
 	cacheTTL time.Duration
 }
 
 func newDataKeyCache(ttl time.Duration) *dataKeyCache {
 	return &dataKeyCache{
-		entries:  make(map[string]dataKeyCacheEntry),
+		byId:     make(map[string]*dataKeyCacheEntry),
+		byName:   make(map[string]*dataKeyCacheEntry),
 		cacheTTL: ttl,
 	}
 }
 
-func (c *dataKeyCache) get(id string) ([]byte, bool) {
-	c.RLock()
-	defer c.RUnlock()
+func (c *dataKeyCache) getById(id string) (*dataKeyCacheEntry, bool) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
 
-	entry, exists := c.entries[id]
+	entry, exists := c.byId[id]
 
 	cacheReadsCounter.With(prometheus.Labels{
-		"hit": strconv.FormatBool(exists),
+		"hit":    strconv.FormatBool(exists),
+		"method": "byId",
 	}).Inc()
 
 	if !exists || entry.expired() {
 		return nil, false
 	}
 
-	return entry.dataKey, true
+	return entry, true
 }
 
-func (c *dataKeyCache) add(id string, dataKey []byte) {
-	c.Lock()
-	defer c.Unlock()
+func (c *dataKeyCache) getByName(name string) (*dataKeyCacheEntry, bool) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
 
-	c.entries[id] = dataKeyCacheEntry{
-		dataKey:    dataKey,
-		expiration: now().Add(c.cacheTTL),
+	entry, exists := c.byName[name]
+
+	cacheReadsCounter.With(prometheus.Labels{
+		"hit":    strconv.FormatBool(exists),
+		"method": "byName",
+	}).Inc()
+
+	if !exists || entry.expired() {
+		return nil, false
 	}
+
+	return entry, true
+}
+
+func (c *dataKeyCache) add(entry *dataKeyCacheEntry) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	entry.expiration = now().Add(c.cacheTTL)
+
+	c.byId[entry.id] = entry
+	c.byName[entry.name] = entry
 }
 
 func (c *dataKeyCache) removeExpired() {
-	c.Lock()
-	defer c.Unlock()
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
 
-	for id, entry := range c.entries {
+	for id, entry := range c.byId {
 		if entry.expired() {
-			delete(c.entries, id)
+			delete(c.byId, id)
+		}
+	}
+
+	for name, entry := range c.byName {
+		if entry.expired() {
+			delete(c.byName, name)
 		}
 	}
 }
 
 func (c *dataKeyCache) flush() {
-	c.Lock()
-	c.entries = make(map[string]dataKeyCacheEntry)
-	c.Unlock()
+	c.mtx.Lock()
+	c.byId = make(map[string]*dataKeyCacheEntry)
+	c.byName = make(map[string]*dataKeyCacheEntry)
+	c.mtx.Unlock()
 }