SecretsManager: Add base encryption manager (#107562)

Co-authored-by: Michael Mandrus <michael.mandrus@grafana.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

pkg/registry/apis/secret/encryption/manager/manager.go

@@ -0,0 +1,403 @@
+package manager
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/usagestats"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher/service"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/kmsproviders"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+const (
+	keyIdDelimiter = '#'
+)
+
+type EncryptionManager struct {
+	tracer     trace.Tracer
+	store      contracts.DataKeyStorage
+	enc        cipher.Cipher
+	cfg        *setting.Cfg
+	usageStats usagestats.Service
+
+	mtx sync.Mutex
+
+	pOnce     sync.Once
+	providers encryption.ProviderMap
+
+	currentProviderID encryption.ProviderID
+
+	log log.Logger
+}
+
+// ProvideEncryptionManager returns an EncryptionManager that uses the OSS KMS providers, along with any additional third-party (e.g. Enterprise) KMS providers
+func ProvideEncryptionManager(
+	tracer trace.Tracer,
+	store contracts.DataKeyStorage,
+	cfg *setting.Cfg,
+	usageStats usagestats.Service,
+	thirdPartyKMS encryption.ProviderMap,
+) (contracts.EncryptionManager, error) {
+	currentProviderID := encryption.ProviderID(cfg.SecretsManagement.EncryptionProvider)
+
+	enc, err := service.NewEncryptionService(tracer, usageStats, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create encryption service: %w", err)
+	}
+
+	s := &EncryptionManager{
+		tracer:            tracer,
+		store:             store,
+		cfg:               cfg,
+		usageStats:        usageStats,
+		enc:               enc,
+		currentProviderID: currentProviderID,
+		log:               log.New("encryption"),
+	}
+
+	if err := s.InitProviders(thirdPartyKMS); err != nil {
+		return nil, err
+	}
+
+	if _, ok := s.providers[currentProviderID]; !ok {
+		return nil, fmt.Errorf("missing configuration for current encryption provider %s", currentProviderID)
+	}
+
+	s.registerUsageMetrics()
+
+	return s, nil
+}
+
+func (s *EncryptionManager) InitProviders(extraProviders encryption.ProviderMap) (err error) {
+	done := false
+	s.pOnce.Do(func() {
+		providers := kmsproviders.GetOSSKMSProviders(s.cfg, s.enc)
+
+		for id, p := range extraProviders {
+			if _, exists := s.providers[id]; exists {
+				err = fmt.Errorf("provider %s already registered", id)
+				return
+			}
+			providers[id] = p
+		}
+
+		s.providers = providers
+		done = true
+	})
+
+	if !done && err == nil {
+		err = fmt.Errorf("providers were already initialized, no action taken")
+	}
+
+	return
+}
+
+func (s *EncryptionManager) registerUsageMetrics() {
+	s.usageStats.RegisterMetricsFunc(func(ctx context.Context) (map[string]any, error) {
+		usageMetrics := make(map[string]any)
+
+		// Current provider
+		kind, err := s.currentProviderID.Kind()
+		if err != nil {
+			return nil, fmt.Errorf("encryptionManager.registerUsageMetrics: %w", err)
+		}
+		usageMetrics[fmt.Sprintf("stats.%s.encryption.current_provider.%s.count", encryption.UsageInsightsPrefix, kind)] = 1
+
+		// Count by kind
+		countByKind := make(map[string]int, len(s.providers))
+		for id := range s.providers {
+			kind, err := id.Kind()
+			if err != nil {
+				return nil, fmt.Errorf("encryptionManager.registerUsageMetrics: %w", err)
+			}
+
+			countByKind[kind]++
+		}
+
+		for kind, count := range countByKind {
+			usageMetrics[fmt.Sprintf("stats.%s.encryption.providers.%s.count", encryption.UsageInsightsPrefix, kind)] = count
+		}
+
+		return usageMetrics, nil
+	})
+}
+
+// TODO: Why do we need to use a global variable for this?
+var b64 = base64.RawStdEncoding
+
+func (s *EncryptionManager) Encrypt(ctx context.Context, namespace string, payload []byte) ([]byte, error) {
+	ctx, span := s.tracer.Start(ctx, "EnvelopeEncryptionManager.Encrypt", trace.WithAttributes(
+		attribute.String("namespace", namespace),
+	))
+	defer span.End()
+
+	var err error
+	defer func() {
+		opsCounter.With(prometheus.Labels{
+			"success":   strconv.FormatBool(err == nil),
+			"operation": OpEncrypt,
+		}).Inc()
+
+		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
+			span.RecordError(err)
+		}
+	}()
+
+	label := encryption.KeyLabel(s.currentProviderID)
+
+	var id string
+	var dataKey []byte
+	id, dataKey, err = s.currentDataKey(ctx, namespace, label)
+	if err != nil {
+		s.log.Error("Failed to get current data key", "error", err, "label", label)
+		return nil, err
+	}
+
+	var encrypted []byte
+	encrypted, err = s.enc.Encrypt(ctx, payload, string(dataKey))
+	if err != nil {
+		s.log.Error("Failed to encrypt secret", "error", err)
+		return nil, err
+	}
+
+	prefix := make([]byte, b64.EncodedLen(len(id))+2)
+	b64.Encode(prefix[1:], []byte(id))
+	prefix[0] = keyIdDelimiter
+	prefix[len(prefix)-1] = keyIdDelimiter
+
+	blob := make([]byte, len(prefix)+len(encrypted))
+	copy(blob, prefix)
+	copy(blob[len(prefix):], encrypted)
+
+	return blob, nil
+}
+
+// currentDataKey looks up for current data key in cache or database by name, and decrypts it.
+// If there's no current data key in cache nor in database it generates a new random data key,
+// and stores it into both the in-memory cache and database (encrypted by the encryption provider).
+func (s *EncryptionManager) currentDataKey(ctx context.Context, namespace string, label string) (string, []byte, error) {
+	ctx, span := s.tracer.Start(ctx, "EnvelopeEncryptionManager.CurrentDataKey", trace.WithAttributes(
+		attribute.String("namespace", namespace),
+		attribute.String("label", label),
+	))
+	defer span.End()
+
+	// We want only one request fetching current data key at time to
+	// avoid the creation of multiple ones in case there's no one existing.
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+
+	// We try to fetch the data key, either from cache or database
+	id, dataKey, err := s.dataKeyByLabel(ctx, namespace, label)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// If no existing data key was found, create a new one
+	if dataKey == nil {
+		id, dataKey, err = s.newDataKey(ctx, namespace, label)
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
+	return id, dataKey, nil
+}
+
+// dataKeyByLabel looks up for data key in cache by label.
+// Otherwise, it fetches it from database, decrypts it and caches it decrypted.
+func (s *EncryptionManager) dataKeyByLabel(ctx context.Context, namespace, label string) (string, []byte, error) {
+	// 1. Get data key from database.
+	dataKey, err := s.store.GetCurrentDataKey(ctx, namespace, label)
+	if err != nil {
+		if errors.Is(err, contracts.ErrDataKeyNotFound) {
+			return "", nil, nil
+		}
+		return "", nil, err
+	}
+
+	// 2.1 Find the encryption provider.
+	provider, exists := s.providers[dataKey.Provider]
+	if !exists {
+		return "", nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
+	}
+
+	// 2.2 Decrypt the data key fetched from the database.
+	decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return dataKey.UID, decrypted, nil
+}
+
+// newDataKey creates a new random data key, encrypts it and stores it into the database.
+func (s *EncryptionManager) newDataKey(ctx context.Context, namespace string, label string) (string, []byte, error) {
+	ctx, span := s.tracer.Start(ctx, "EnvelopeEncryptionManager.NewDataKey", trace.WithAttributes(
+		attribute.String("namespace", namespace),
+		attribute.String("label", label),
+	))
+	defer span.End()
+
+	// 1. Create new data key.
+	dataKey, err := newRandomDataKey()
+	if err != nil {
+		return "", nil, err
+	}
+
+	// 2.1 Find the encryption provider.
+	provider, exists := s.providers[s.currentProviderID]
+	if !exists {
+		return "", nil, fmt.Errorf("could not find encryption provider '%s'", s.currentProviderID)
+	}
+
+	// 2.2 Encrypt the data key.
+	encrypted, err := provider.Encrypt(ctx, dataKey)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// 3. Store its encrypted value into the DB.
+	id := util.GenerateShortUID()
+
+	dbDataKey := contracts.SecretDataKey{
+		Active:        true,
+		UID:           id,
+		Namespace:     namespace,
+		Provider:      s.currentProviderID,
+		EncryptedData: encrypted,
+		Label:         label,
+	}
+
+	err = s.store.CreateDataKey(ctx, &dbDataKey)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return id, dataKey, nil
+}
+
+func newRandomDataKey() ([]byte, error) {
+	rawDataKey := make([]byte, 16)
+	_, err := rand.Read(rawDataKey)
+	if err != nil {
+		return nil, err
+	}
+	return rawDataKey, nil
+}
+
+func (s *EncryptionManager) Decrypt(ctx context.Context, namespace string, payload []byte) ([]byte, error) {
+	ctx, span := s.tracer.Start(ctx, "EnvelopeEncryptionManager.Decrypt", trace.WithAttributes(
+		attribute.String("namespace", namespace),
+	))
+	defer span.End()
+
+	var err error
+	defer func() {
+		opsCounter.With(prometheus.Labels{
+			"success":   strconv.FormatBool(err == nil),
+			"operation": OpDecrypt,
+		}).Inc()
+
+		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
+			span.RecordError(err)
+
+			s.log.FromContext(ctx).Error("Failed to decrypt secret", "error", err)
+		}
+	}()
+
+	if len(payload) == 0 {
+		err = fmt.Errorf("unable to decrypt empty payload")
+		return nil, err
+	}
+
+	payload = payload[1:]
+	endOfKey := bytes.Index(payload, []byte{keyIdDelimiter})
+	if endOfKey == -1 {
+		err = fmt.Errorf("could not find valid key id in encrypted payload")
+		return nil, err
+	}
+	b64Key := payload[:endOfKey]
+	payload = payload[endOfKey+1:]
+	keyId := make([]byte, b64.DecodedLen(len(b64Key)))
+	_, err = b64.Decode(keyId, b64Key)
+	if err != nil {
+		return nil, err
+	}
+
+	dataKey, err := s.dataKeyById(ctx, namespace, string(keyId))
+	if err != nil {
+		s.log.FromContext(ctx).Error("Failed to lookup data key by id", "id", string(keyId), "error", err)
+		return nil, err
+	}
+
+	var decrypted []byte
+	decrypted, err = s.enc.Decrypt(ctx, payload, string(dataKey))
+
+	return decrypted, err
+}
+
+func (s *EncryptionManager) GetDecryptedValue(ctx context.Context, namespace string, sjd map[string][]byte, key, fallback string) string {
+	if value, ok := sjd[key]; ok {
+		decryptedData, err := s.Decrypt(ctx, namespace, value)
+		if err != nil {
+			return fallback
+		}
+
+		return string(decryptedData)
+	}
+
+	return fallback
+}
+
+// dataKeyById looks up for data key in the database and returns it decrypted.
+func (s *EncryptionManager) dataKeyById(ctx context.Context, namespace, id string) ([]byte, error) {
+	ctx, span := s.tracer.Start(ctx, "EnvelopeEncryptionManager.GetDataKey", trace.WithAttributes(
+		attribute.String("namespace", namespace),
+		attribute.String("id", id),
+	))
+	defer span.End()
+
+	// 1. Get encrypted data key from database.
+	dataKey, err := s.store.GetDataKey(ctx, namespace, id)
+	if err != nil {
+		return nil, err
+	}
+
+	// 2.1. Find the encryption provider.
+	provider, exists := s.providers[dataKey.Provider]
+	if !exists {
+		return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
+	}
+
+	// 2.2. Decrypt the data key.
+	decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
+	if err != nil {
+		return nil, err
+	}
+
+	return decrypted, nil
+}
+
+func (s *EncryptionManager) GetProviders() encryption.ProviderMap {
+	return s.providers
+}

pkg/registry/apis/secret/encryption/manager/manager_test.go

@@ -0,0 +1,448 @@
+package manager
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+	"gopkg.in/ini.v1"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/usagestats"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/storage/secret/database"
+	encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
+	"github.com/grafana/grafana/pkg/storage/secret/migrator"
+	"github.com/grafana/grafana/pkg/tests/testsuite"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+func TestMain(m *testing.M) {
+	testsuite.Run(m)
+}
+
+func TestEncryptionService_EnvelopeEncryption(t *testing.T) {
+	svc := setupTestService(t)
+	ctx := context.Background()
+	namespace := "test-namespace"
+
+	t.Run("encrypting should create DEK", func(t *testing.T) {
+		plaintext := []byte("very secret string")
+
+		encrypted, err := svc.Encrypt(context.Background(), namespace, plaintext)
+		require.NoError(t, err)
+
+		decrypted, err := svc.Decrypt(context.Background(), namespace, encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, plaintext, decrypted)
+
+		keys, err := svc.store.GetAllDataKeys(ctx, namespace)
+		require.NoError(t, err)
+		assert.Equal(t, len(keys), 1)
+	})
+
+	t.Run("encrypting another secret should use the same DEK", func(t *testing.T) {
+		plaintext := []byte("another very secret string")
+
+		encrypted, err := svc.Encrypt(context.Background(), namespace, plaintext)
+		require.NoError(t, err)
+
+		decrypted, err := svc.Decrypt(context.Background(), namespace, encrypted)
+		require.NoError(t, err)
+		assert.Equal(t, plaintext, decrypted)
+
+		keys, err := svc.store.GetAllDataKeys(ctx, namespace)
+		require.NoError(t, err)
+		assert.Equal(t, len(keys), 1)
+	})
+
+	t.Run("usage stats should be registered", func(t *testing.T) {
+		reports, err := svc.usageStats.GetUsageReport(context.Background())
+		require.NoError(t, err)
+
+		assert.Equal(t, 1, reports.Metrics["stats.secrets_manager.encryption.current_provider.secretKey.count"])
+		assert.Equal(t, 1, reports.Metrics["stats.secrets_manager.encryption.providers.secretKey.count"])
+	})
+}
+
+func TestEncryptionService_DataKeys(t *testing.T) {
+	// Initialize data key storage with a fake db
+	testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
+	features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
+	tracer := noop.NewTracerProvider().Tracer("test")
+	store, err := encryptionstorage.ProvideDataKeyStorage(database.ProvideDatabase(testDB, tracer), tracer, features)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	namespace := "test-namespace"
+
+	dataKey := &contracts.SecretDataKey{
+		UID:           util.GenerateShortUID(),
+		Label:         "test1",
+		Active:        true,
+		Provider:      "test",
+		EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
+		Namespace:     namespace,
+	}
+
+	t.Run("querying for a DEK that does not exist", func(t *testing.T) {
+		res, err := store.GetDataKey(ctx, namespace, dataKey.UID)
+		assert.ErrorIs(t, contracts.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+
+	t.Run("creating an active DEK", func(t *testing.T) {
+		err := store.CreateDataKey(ctx, dataKey)
+		require.NoError(t, err)
+
+		res, err := store.GetDataKey(ctx, namespace, dataKey.UID)
+		require.NoError(t, err)
+		assert.Equal(t, dataKey.EncryptedData, res.EncryptedData)
+		assert.Equal(t, dataKey.Provider, res.Provider)
+		assert.Equal(t, dataKey.Label, res.Label)
+		assert.Equal(t, dataKey.UID, res.UID)
+		assert.True(t, dataKey.Active)
+
+		current, err := store.GetCurrentDataKey(ctx, namespace, dataKey.Label)
+		require.NoError(t, err)
+		assert.Equal(t, dataKey.EncryptedData, current.EncryptedData)
+		assert.Equal(t, dataKey.Provider, current.Provider)
+		assert.Equal(t, dataKey.Label, current.Label)
+		assert.Equal(t, dataKey.UID, current.UID)
+		assert.True(t, current.Active)
+	})
+
+	t.Run("creating an inactive DEK", func(t *testing.T) {
+		k := &contracts.SecretDataKey{
+			UID:           util.GenerateShortUID(),
+			Namespace:     namespace,
+			Active:        false,
+			Label:         "test2",
+			Provider:      "test",
+			EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
+		}
+
+		err := store.CreateDataKey(ctx, k)
+		require.Error(t, err)
+
+		res, err := store.GetDataKey(ctx, namespace, k.UID)
+		assert.Equal(t, contracts.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+
+	t.Run("deleting DEK when no id provided must fail", func(t *testing.T) {
+		beforeDelete, err := store.GetAllDataKeys(ctx, namespace)
+		require.NoError(t, err)
+		err = store.DeleteDataKey(ctx, namespace, "")
+		require.Error(t, err)
+
+		afterDelete, err := store.GetAllDataKeys(ctx, namespace)
+		require.NoError(t, err)
+		assert.Equal(t, beforeDelete, afterDelete)
+	})
+
+	t.Run("deleting a DEK", func(t *testing.T) {
+		err := store.DeleteDataKey(ctx, namespace, dataKey.UID)
+		require.NoError(t, err)
+
+		res, err := store.GetDataKey(ctx, namespace, dataKey.UID)
+		assert.Equal(t, contracts.ErrDataKeyNotFound, err)
+		assert.Nil(t, res)
+	})
+}
+
+func TestEncryptionService_UseCurrentProvider(t *testing.T) {
+	t.Run("When encryption_provider is not specified explicitly, should use 'secretKey' as a current provider", func(t *testing.T) {
+		svc := setupTestService(t)
+		assert.Equal(t, encryption.ProviderID("secretKey.v1"), svc.currentProviderID)
+	})
+
+	t.Run("Should use encrypt/decrypt methods of the current encryption provider", func(t *testing.T) {
+		rawCfg := `
+		[secrets_manager.encryption.fakeProvider.v1]
+		`
+
+		raw, err := ini.Load([]byte(rawCfg))
+		require.NoError(t, err)
+
+		cfg := &setting.Cfg{
+			Raw: raw,
+			SecretsManagement: setting.SecretsManagerSettings{
+				SecretKey:          "sdDkslslld",
+				EncryptionProvider: "secretKey.v1",
+			},
+		}
+
+		features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
+		testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
+		tracer := noop.NewTracerProvider().Tracer("test")
+		encryptionStore, err := encryptionstorage.ProvideDataKeyStorage(database.ProvideDatabase(testDB, tracer), tracer, features)
+		require.NoError(t, err)
+
+		encMgr, err := ProvideEncryptionManager(
+			tracer,
+			encryptionStore,
+			cfg,
+			&usagestats.UsageStatsMock{T: t},
+			encryption.ProvideThirdPartyProviderMap(),
+		)
+		require.NoError(t, err)
+
+		encryptionManager := encMgr.(*EncryptionManager)
+
+		//override default provider with fake, and register the fake separately
+		fake := &fakeProvider{}
+		encryptionManager.providers[encryption.ProviderID("fakeProvider.v1")] = fake
+		encryptionManager.currentProviderID = "fakeProvider.v1"
+
+		namespace := "test-namespace"
+		encrypted, _ := encryptionManager.Encrypt(context.Background(), namespace, []byte{})
+		assert.True(t, fake.encryptCalled)
+		assert.False(t, fake.decryptCalled)
+
+		// encryption manager tries to find a DEK in a cache first before calling provider's decrypt
+		// to bypass the cache, we set up one more secrets service to test decrypting
+		svcDecryptMgr, err := ProvideEncryptionManager(
+			tracer,
+			encryptionStore,
+			cfg,
+			&usagestats.UsageStatsMock{T: t},
+			encryption.ProvideThirdPartyProviderMap(),
+		)
+		require.NoError(t, err)
+
+		svcDecrypt := svcDecryptMgr.(*EncryptionManager)
+		svcDecrypt.providers[encryption.ProviderID("fakeProvider.v1")] = fake
+		svcDecrypt.currentProviderID = "fakeProvider.v1"
+
+		_, _ = svcDecrypt.Decrypt(context.Background(), namespace, encrypted)
+		assert.True(t, fake.decryptCalled, "fake provider's decrypt should be called")
+	})
+}
+
+type fakeProvider struct {
+	encryptCalled bool
+	decryptCalled bool
+}
+
+func (p *fakeProvider) Encrypt(_ context.Context, _ []byte) ([]byte, error) {
+	p.encryptCalled = true
+	return []byte{}, nil
+}
+
+func (p *fakeProvider) Decrypt(_ context.Context, _ []byte) ([]byte, error) {
+	p.decryptCalled = true
+	return []byte{}, nil
+}
+
+func TestEncryptionService_Decrypt(t *testing.T) {
+	ctx := context.Background()
+	namespace := "test-namespace"
+
+	t.Run("empty payload should fail", func(t *testing.T) {
+		svc := setupTestService(t)
+		_, err := svc.Decrypt(context.Background(), namespace, []byte(""))
+		require.Error(t, err)
+
+		assert.Equal(t, "unable to decrypt empty payload", err.Error())
+	})
+
+	t.Run("ee encrypted payload with ee enabled should work", func(t *testing.T) {
+		svc := setupTestService(t)
+		ciphertext, err := svc.Encrypt(ctx, namespace, []byte("grafana"))
+		require.NoError(t, err)
+
+		plaintext, err := svc.Decrypt(ctx, namespace, ciphertext)
+		assert.NoError(t, err)
+		assert.Equal(t, []byte("grafana"), plaintext)
+	})
+}
+
+func TestIntegration_SecretsService(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+	someData := []byte(`some-data`)
+	namespace := "test-namespace"
+
+	tcs := map[string]func(*testing.T, db.DB, contracts.EncryptionManager){
+		"regular": func(t *testing.T, _ db.DB, svc contracts.EncryptionManager) {
+			// We encrypt some data normally, no transactions implied.
+			_, err := svc.Encrypt(ctx, namespace, someData)
+			require.NoError(t, err)
+		},
+		"within successful InTransaction": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NoError(t, store.InTransaction(ctx, func(ctx context.Context) error {
+				// We encrypt some data within a transaction that shares the db session.
+				_, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// And the transition succeeds.
+				return nil
+			}))
+		},
+		"within unsuccessful InTransaction": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NotNil(t, store.InTransaction(ctx, func(ctx context.Context) error {
+				// We encrypt some data within a transaction that shares the db session.
+				_, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// But the transaction fails.
+				return errors.New("error")
+			}))
+		},
+		"within unsuccessful InTransaction (plus forced db fetch)": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NotNil(t, store.InTransaction(ctx, func(ctx context.Context) error {
+				// We encrypt some data within a transaction that shares the db session.
+				encrypted, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// At this point the data key is not cached yet because
+				// the transaction haven't been committed yet,
+				// and won't, so we do a decrypt operation within the
+				// transaction to force the data key to be
+				// (potentially) cached (it shouldn't to prevent issues).
+				decrypted, err := svc.Decrypt(ctx, namespace, encrypted)
+				require.NoError(t, err)
+				assert.Equal(t, someData, decrypted)
+
+				// But the transaction fails.
+				return errors.New("error")
+			}))
+		},
+		"within successful WithTransactionalDbSession": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NoError(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
+				// We encrypt some data within a transaction that does not share the db session.
+				_, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// And the transition succeeds.
+				return nil
+			}))
+		},
+		"within unsuccessful WithTransactionalDbSession": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NotNil(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
+				// We encrypt some data within a transaction that does not share the db session.
+				_, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// But the transaction fails.
+				return errors.New("error")
+			}))
+		},
+		"within unsuccessful WithTransactionalDbSession (plus forced db fetch)": func(t *testing.T, store db.DB, svc contracts.EncryptionManager) {
+			require.NotNil(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
+				// We encrypt some data within a transaction that does not share the db session.
+				encrypted, err := svc.Encrypt(ctx, namespace, someData)
+				require.NoError(t, err)
+
+				// At this point the data key is not cached yet because
+				// the transaction haven't been committed yet,
+				// and won't, so we do a decrypt operation within the
+				// transaction to force the data key to be
+				// (potentially) cached (it shouldn't to prevent issues).
+				decrypted, err := svc.Decrypt(ctx, namespace, encrypted)
+				require.NoError(t, err)
+				assert.Equal(t, someData, decrypted)
+
+				// But the transaction fails.
+				return errors.New("error")
+			}))
+		},
+	}
+
+	for name, tc := range tcs {
+		t.Run(name, func(t *testing.T) {
+			testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
+			tracer := noop.NewTracerProvider().Tracer("test")
+
+			features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
+			defaultKey := "SdlklWklckeLS"
+
+			cfg := &setting.Cfg{
+				SecretsManagement: setting.SecretsManagerSettings{
+					SecretKey:          defaultKey,
+					EncryptionProvider: "secretKey.v1",
+				},
+			}
+			store, err := encryptionstorage.ProvideDataKeyStorage(database.ProvideDatabase(testDB, tracer), tracer, features)
+			require.NoError(t, err)
+
+			usageStats := &usagestats.UsageStatsMock{T: t}
+
+			svc, err := ProvideEncryptionManager(
+				tracer,
+				store,
+				cfg,
+				usageStats,
+				encryption.ProvideThirdPartyProviderMap(),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			namespace := "test-namespace"
+
+			// Here's what actually matters and varies on each test: look at the test case name.
+			//
+			// For historical reasons, and in an old implementation, when a successful encryption
+			// operation happened within an unsuccessful transaction, the data key was used to be
+			// cached in memory for the next encryption operations, which caused some data to be
+			// encrypted with a data key that haven't actually been persisted into the database.
+			tc(t, testDB, svc)
+			// Therefore, the data encrypted after this point, become unrecoverable after a restart.
+			// So, the different test cases here are there to prevent that from happening again
+			// in the future, whatever it is what happens.
+
+			// So, we proceed with an encryption operation:
+			toEncrypt := []byte(`data-to-encrypt`)
+			encrypted, err := svc.Encrypt(ctx, namespace, toEncrypt)
+			require.NoError(t, err)
+
+			// And then, we MUST still be able to decrypt the previously encrypted data:
+			decrypted, err := svc.Decrypt(ctx, namespace, encrypted)
+			require.NoError(t, err)
+			assert.Equal(t, toEncrypt, decrypted)
+		})
+	}
+}
+
+func TestEncryptionService_ReInitReturnsError(t *testing.T) {
+	svc := setupTestService(t)
+	err := svc.InitProviders(encryption.ProviderMap{
+		"fakeProvider.v1": &fakeProvider{},
+	})
+	require.Error(t, err)
+}
+
+func TestEncryptionService_ThirdPartyProviders(t *testing.T) {
+	cfg := &setting.Cfg{
+		SecretsManagement: setting.SecretsManagerSettings{
+			SecretKey:          "SdlklWklckeLS",
+			EncryptionProvider: "secretKey.v1",
+		},
+	}
+
+	svc, err := ProvideEncryptionManager(
+		nil,
+		nil,
+		cfg,
+		&usagestats.UsageStatsMock{},
+		encryption.ProviderMap{
+			"fakeProvider.v1": &fakeProvider{},
+		},
+	)
+	require.NoError(t, err)
+
+	encMgr := svc.(*EncryptionManager)
+	require.Len(t, encMgr.providers, 2)
+	require.Contains(t, encMgr.providers, encryption.ProviderID("fakeProvider.v1"))
+}

pkg/registry/apis/secret/encryption/manager/metrics.go

@@ -0,0 +1,51 @@
+package manager
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
+)
+
+const (
+	OpEncrypt = "encrypt"
+	OpDecrypt = "decrypt"
+	subsystem = "encryption_manager"
+)
+
+// TODO: Add timing metrics after the encryption module cleanup
+var (
+	opsCounter = metricutil.NewCounterVecStartingAtZero(
+		prometheus.CounterOpts{
+			Namespace: metrics.ExporterName,
+			Subsystem: subsystem,
+			Name:      "encryption_ops_total",
+			Help:      "A counter for encryption operations",
+		},
+		[]string{"success", "operation"},
+		map[string][]string{
+			"success":   {"true", "false"},
+			"operation": {OpEncrypt, OpDecrypt},
+		},
+	)
+	cacheReadsCounter = metricutil.NewCounterVecStartingAtZero(
+		prometheus.CounterOpts{
+			Namespace: metrics.ExporterName,
+			Subsystem: subsystem,
+			Name:      "encryption_cache_reads_total",
+			Help:      "A counter for encryption cache reads",
+		},
+		[]string{"hit", "method"},
+		map[string][]string{
+			"hit":    {"true", "false"},
+			"method": {"byId", "byName"},
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(
+		opsCounter,
+		cacheReadsCounter,
+	)
+}

pkg/registry/apis/secret/encryption/manager/test_helpers.go

@@ -0,0 +1,49 @@
+package manager
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+
+	"github.com/grafana/grafana/pkg/infra/usagestats"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/storage/secret/database"
+	encryptionstorage "github.com/grafana/grafana/pkg/storage/secret/encryption"
+	"github.com/grafana/grafana/pkg/storage/secret/migrator"
+)
+
+func setupTestService(tb testing.TB) *EncryptionManager {
+	tb.Helper()
+
+	testDB := sqlstore.NewTestStore(tb, sqlstore.WithMigrator(migrator.New()))
+	tracer := noop.NewTracerProvider().Tracer("test")
+	database := database.ProvideDatabase(testDB, tracer)
+
+	features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
+	defaultKey := "SdlklWklckeLS"
+	cfg := &setting.Cfg{
+		SecretsManagement: setting.SecretsManagerSettings{
+			SecretKey:          defaultKey,
+			EncryptionProvider: "secretKey.v1",
+		},
+	}
+	store, err := encryptionstorage.ProvideDataKeyStorage(database, tracer, features)
+	require.NoError(tb, err)
+
+	usageStats := &usagestats.UsageStatsMock{T: tb}
+
+	encMgr, err := ProvideEncryptionManager(
+		tracer,
+		store,
+		cfg,
+		usageStats,
+		encryption.ProvideThirdPartyProviderMap(),
+	)
+	require.NoError(tb, err)
+
+	return encMgr.(*EncryptionManager)
+}