Secrets service: Allow decrypt through gRPC connection (#108365)

pkg/registry/apis/provisioning/secrets/mocks/decrypt_service_mock.go

@@ -22,6 +22,51 @@ func (_m *MockDecryptService) EXPECT() *MockDecryptService_Expecter {
 	return &MockDecryptService_Expecter{mock: &_m.Mock}
 }
 
+// Close provides a mock function with no fields
+func (_m *MockDecryptService) Close() error {
+	ret := _m.Called()
+
+	if len(ret) == 0 {
+		panic("no return value specified for Close")
+	}
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// MockDecryptService_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
+type MockDecryptService_Close_Call struct {
+	*mock.Call
+}
+
+// Close is a helper method to define mock.On call
+func (_e *MockDecryptService_Expecter) Close() *MockDecryptService_Close_Call {
+	return &MockDecryptService_Close_Call{Call: _e.mock.On("Close")}
+}
+
+func (_c *MockDecryptService_Close_Call) Run(run func()) *MockDecryptService_Close_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run()
+	})
+	return _c
+}
+
+func (_c *MockDecryptService_Close_Call) Return(_a0 error) *MockDecryptService_Close_Call {
+	_c.Call.Return(_a0)
+	return _c
+}
+
+func (_c *MockDecryptService_Close_Call) RunAndReturn(run func() error) *MockDecryptService_Close_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
 // Decrypt provides a mock function with given fields: ctx, namespace, names
 func (_m *MockDecryptService) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
 	_va := make([]interface{}, len(names))

pkg/registry/apis/provisioning/secrets/repository.go

@@ -11,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	legacysecrets "github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 func ProvideRepositorySecrets(
@@ -18,8 +19,9 @@ func ProvideRepositorySecrets(
 	legacySecretsSvc legacysecrets.Service,
 	secretsSvc contracts.SecureValueClient,
 	decryptSvc secret.DecryptService,
+	cfg *setting.Cfg,
 ) RepositorySecrets {
-	return NewRepositorySecrets(features, NewSecretsService(secretsSvc, decryptSvc), NewSingleTenant(legacySecretsSvc))
+	return NewRepositorySecrets(features, NewSecretsService(secretsSvc, decryptSvc, cfg), NewSingleTenant(legacySecretsSvc))
 }
 
 //go:generate mockery --name RepositorySecrets --structname MockRepositorySecrets --inpackage --filename repository_secrets_mock.go --with-expecter

pkg/registry/apis/provisioning/secrets/secret.go

@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/registry/apis/secret"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/setting"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -31,12 +32,14 @@ var _ Service = (*secretsService)(nil)
 type secretsService struct {
 	secureValues SecureValueClient
 	decryptSvc   secret.DecryptService
+	cfg          *setting.Cfg
 }
 
-func NewSecretsService(secretsSvc SecureValueClient, decryptSvc secret.DecryptService) Service {
+func NewSecretsService(secretsSvc SecureValueClient, decryptSvc secret.DecryptService, cfg *setting.Cfg) Service {
 	return &secretsService{
 		secureValues: secretsSvc,
 		decryptSvc:   decryptSvc,
+		cfg:          cfg,
 	}
 }
 
@@ -74,6 +77,11 @@ func (s *secretsService) Encrypt(ctx context.Context, namespace, name string, da
 		return result.GetName(), nil
 	}
 
+	decrypters := []string{svcName}
+	if s.cfg.SecretsManagement.DecryptGrafanaServiceName != "" {
+		decrypters = append(decrypters, s.cfg.SecretsManagement.DecryptGrafanaServiceName)
+	}
+
 	// Create the secret directly as unstructured
 	secret := &unstructured.Unstructured{
 		Object: map[string]interface{}{
@@ -86,7 +94,7 @@ func (s *secretsService) Encrypt(ctx context.Context, namespace, name string, da
 			"spec": map[string]interface{}{
 				"description": "provisioning: " + name,
 				"value":       data,
-				"decrypters":  []string{svcName},
+				"decrypters":  decrypters,
 			},
 		},
 	}

pkg/registry/apis/provisioning/secrets/secret_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/secrets/mocks"
 	"github.com/grafana/grafana/pkg/registry/apis/secret"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -50,7 +51,7 @@ func TestNewSecretsService(t *testing.T) {
 	mockSecretsSvc := NewMockSecureValueClient(t)
 	mockDecryptSvc := &mocks.MockDecryptService{}
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 
 	assert.NotNil(t, svc)
 	assert.IsType(t, &secretsService{}, svc)
@@ -208,7 +209,7 @@ func TestSecretsService_Encrypt(t *testing.T) {
 
 			tt.setupMocks(mockSecretsSvc, mockDecryptSvc, mockResourceInterface)
 
-			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 
 			ctx := context.Background()
 
@@ -232,7 +233,7 @@ func TestSecretsService_Encrypt_ClientError(t *testing.T) {
 	// Setup client to return error
 	mockSecretsSvc.EXPECT().Client(mock.Anything, "test-namespace").Return(nil, errors.New("client error"))
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 
 	ctx := context.Background()
 
@@ -331,7 +332,7 @@ func TestSecretsService_Decrypt(t *testing.T) {
 
 			tt.setupMocks(mockSecretsSvc, mockDecryptSvc)
 
-			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 
 			ctx := context.Background()
 
@@ -368,7 +369,7 @@ func TestSecretsService_Decrypt_ServiceIdentityContext(t *testing.T) {
 		"test-secret": mockResult,
 	}, nil)
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 
 	ctx := context.Background()
 	result, err := svc.Decrypt(ctx, "test-namespace", "test-secret")
@@ -430,7 +431,7 @@ func TestSecretsService_Delete(t *testing.T) {
 
 			tt.setupMocks(mockSecretsSvc, mockDecryptSvc, mockResourceInterface)
 
-			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+			svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 			ctx := context.Background()
 
 			err := svc.Delete(ctx, tt.namespace, tt.secretName)
@@ -515,7 +516,7 @@ func TestSecretsService_Encrypt_WithK8sNotFoundError(t *testing.T) {
 	}
 	mockResourceInterface.createErr = nil
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 	ctx := context.Background()
 
 	result, err := svc.Encrypt(ctx, "test-namespace", "test-secret", "secret-data")
@@ -536,7 +537,7 @@ func TestSecretsService_Delete_WithK8sNotFoundError(t *testing.T) {
 	k8sNotFoundErr := apierrors.NewNotFound(schema.GroupResource{Group: "secret.grafana.app", Resource: "securevalues"}, "test-secret")
 	mockResourceInterface.deleteErr = k8sNotFoundErr
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 	ctx := context.Background()
 
 	err := svc.Delete(ctx, "test-namespace", "test-secret")
@@ -557,7 +558,7 @@ func TestSecretsService_Delete_WithGrafanaNotFoundError(t *testing.T) {
 	// Mock Delete call to return Grafana not found error
 	mockResourceInterface.deleteErr = contracts.ErrSecureValueNotFound
 
-	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc)
+	svc := NewSecretsService(mockSecretsSvc, mockDecryptSvc, &setting.Cfg{})
 	ctx := context.Background()
 
 	err := svc.Delete(ctx, "test-namespace", "test-secret")

pkg/registry/apis/provisioning/secrets/secure_value_client_mock.go

@@ -5,8 +5,9 @@ package secrets
 import (
 	context "context"
 
-	mock "github.com/stretchr/testify/mock"
 	dynamic "k8s.io/client-go/dynamic"
+
+	mock "github.com/stretchr/testify/mock"
 )
 
 // MockSecureValueClient is an autogenerated mock type for the SecureValueClient type

pkg/registry/apis/secret/contracts/decrypt.go

@@ -8,6 +8,9 @@ import (
 	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
 )
 
+// HeaderGrafanaServiceIdentityName is used to pass the service identity in the gRPC request metadata.
+const HeaderGrafanaServiceIdentityName = "X-Grafana-Service-Identity-Name"
+
 var (
 	ErrDecryptNotFound      = errors.New("not found")
 	ErrDecryptNotAuthorized = errors.New("not authorized")
@@ -27,6 +30,7 @@ type DecryptAuthorizer interface {
 // DecryptService is the inferface for the decrypt service.
 type DecryptService interface {
 	Decrypt(ctx context.Context, namespace string, names ...string) (map[string]DecryptResult, error)
+	Close() error
 }
 
 // DecryptResult is the (union) result of a decryption operation.

pkg/registry/apis/secret/decrypt/grpc_client.go

@@ -0,0 +1,178 @@
+package decrypt
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/fullstorydev/grpchan"
+	authnlib "github.com/grafana/authlib/authn"
+	claims "github.com/grafana/authlib/types"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+
+	decryptv1beta1 "github.com/grafana/grafana/apps/secret/decrypt/v1beta1"
+	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+)
+
+type GRPCDecryptClient struct {
+	conn   *grpc.ClientConn
+	client decryptv1beta1.SecureValueDecrypterClient
+}
+
+var _ contracts.DecryptService = &GRPCDecryptClient{}
+
+type TLSConfig struct {
+	UseTLS             bool
+	CertFile           string
+	KeyFile            string
+	CAFile             string
+	ServerName         string
+	InsecureSkipVerify bool
+}
+
+func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, namespace, address string) (*GRPCDecryptClient, error) {
+	return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, namespace, address, TLSConfig{})
+}
+
+func NewGRPCDecryptClientWithTLS(
+	tokenExchanger authnlib.TokenExchanger,
+	tracer trace.Tracer,
+	namespace string,
+	address string,
+	tlsConfig TLSConfig,
+) (*GRPCDecryptClient, error) {
+	var opts []grpc.DialOption
+	if tlsConfig.UseTLS {
+		creds, err := createTLSCredentials(tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to setup TLS: %w", err)
+		}
+
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+
+	conn, err := grpc.NewClient(address, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)
+	}
+
+	tokenExchangerInterceptor := authnlib.NewGrpcClientInterceptor(
+		tokenExchanger,
+		authnlib.WithClientInterceptorTracer(tracer),
+		authnlib.WithClientInterceptorNamespace(namespace),
+		authnlib.WithClientInterceptorAudience([]string{secretv1beta1.APIGroup}),
+	)
+
+	clientConn := grpchan.InterceptClientConn(
+		conn,
+		tokenExchangerInterceptor.UnaryClientInterceptor,
+		tokenExchangerInterceptor.StreamClientInterceptor,
+	)
+
+	return &GRPCDecryptClient{
+		conn:   conn,
+		client: decryptv1beta1.NewSecureValueDecrypterClient(clientConn),
+	}, nil
+}
+
+func createTLSCredentials(config TLSConfig) (credentials.TransportCredentials, error) {
+	tlsConfig := &tls.Config{}
+
+	if config.CAFile != "" {
+		caCert, err := os.ReadFile(config.CAFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA: %w", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to append CA")
+		}
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	if config.CertFile != "" && config.KeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load client certificate: %w", err)
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	if config.ServerName != "" {
+		tlsConfig.ServerName = config.ServerName
+	}
+
+	if config.InsecureSkipVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
+
+	return credentials.NewTLS(tlsConfig), nil
+}
+
+func (g *GRPCDecryptClient) Close() error {
+	if g.conn != nil {
+		return g.conn.Close()
+	}
+	return nil
+}
+
+func (g *GRPCDecryptClient) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
+	authInfo, ok := claims.AuthInfoFrom(ctx)
+	if !ok {
+		return nil, errors.New("missing auth info in context")
+	}
+
+	// Up until here the identity is the one set by the internal service, but when the request goes out to the gRPC server,
+	// the aggregator will use the access token which contains a different service identity for grafana as a whole.
+	// This is used for logging purposes only.
+	serviceIdentityList, ok := authInfo.GetExtra()[authnlib.ServiceIdentityKey]
+	if !ok || len(serviceIdentityList) != 1 {
+		return nil, errors.New("invalid service identity in auth info")
+	}
+
+	serviceIdentity := serviceIdentityList[0]
+	if len(serviceIdentity) == 0 {
+		return nil, errors.New("empty service identity in auth info")
+	}
+
+	req := &decryptv1beta1.SecureValueDecryptRequest{
+		Namespace: namespace,
+		Names:     names,
+	}
+
+	// Decryption will still use the service identity from the auth token,
+	// but we also pass the service identity from the request metadata for auditing purposes.
+	md := metadata.New(map[string]string{
+		contracts.HeaderGrafanaServiceIdentityName: serviceIdentity,
+	})
+	ctx = metadata.NewOutgoingContext(ctx, md)
+
+	resp, err := g.client.DecryptSecureValues(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("grpc decrypt failed: %w", err)
+	}
+
+	results := make(map[string]contracts.DecryptResult, len(resp.GetDecryptedValues()))
+
+	for name, result := range resp.GetDecryptedValues() {
+		if result.GetErrorMessage() != "" {
+			results[name] = contracts.NewDecryptResultErr(errors.New(result.GetErrorMessage()))
+		} else {
+			exposedSecureValue := secretv1beta1.NewExposedSecureValue(result.GetValue())
+			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+		}
+	}
+
+	return results, nil
+}

pkg/registry/apis/secret/decrypt/local_client.go

@@ -0,0 +1,39 @@
+package decrypt
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+)
+
+type LocalDecryptClient struct {
+	decryptStorage contracts.DecryptStorage
+}
+
+var _ contracts.DecryptService = &LocalDecryptClient{}
+
+func NewLocalDecryptClient(decryptStorage contracts.DecryptStorage) (*LocalDecryptClient, error) {
+	return &LocalDecryptClient{
+		decryptStorage: decryptStorage,
+	}, nil
+}
+
+func (c *LocalDecryptClient) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
+	results := make(map[string]contracts.DecryptResult, len(names))
+
+	for _, name := range names {
+		exposedSecureValue, err := c.decryptStorage.Decrypt(ctx, xkube.Namespace(namespace), name)
+		if err != nil {
+			results[name] = contracts.NewDecryptResultErr(err)
+		} else {
+			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+		}
+	}
+
+	return results, nil
+}
+
+func (c *LocalDecryptClient) Close() error {
+	return nil
+}

pkg/registry/apis/secret/decrypt/service.go

@@ -1,33 +1,78 @@
 package decrypt
 
 import (
-	"context"
+	"fmt"
+
+	authnlib "github.com/grafana/authlib/authn"
+	claims "github.com/grafana/authlib/types"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/grafana/grafana/pkg/services/authn/grpcutils"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
-type OSSDecryptService struct {
-	decryptStore contracts.DecryptStorage
-}
+func ProvideDecryptService(cfg *setting.Cfg, tracer trace.Tracer, decryptStorage contracts.DecryptStorage) (contracts.DecryptService, error) {
+	switch cfg.SecretsManagement.DecryptServerType {
+	case "grpc":
+		grpcClientConfig := grpcutils.ReadGrpcClientConfig(cfg)
 
-func ProvideDecryptService(decryptStore contracts.DecryptStorage) contracts.DecryptService {
-	return &OSSDecryptService{
-		decryptStore: decryptStore,
-	}
-}
+		if cfg.SecretsManagement.DecryptServerAddress == "" {
+			return nil, fmt.Errorf("decrypt_server_address is required when decrypt_server_type is grpc")
+		}
 
-func (d *OSSDecryptService) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
-	results := make(map[string]contracts.DecryptResult, len(names))
+		if grpcClientConfig.Token == "" || grpcClientConfig.TokenExchangeURL == "" {
+			return nil, fmt.Errorf("grpc_client_authentication.token and grpc_client_authentication.token_exchange_url are required when secrets_manager.decrypt_server_type is grpc")
+		}
 
-	for _, name := range names {
-		exposedSecureValue, err := d.decryptStore.Decrypt(ctx, xkube.Namespace(namespace), name)
+		nsInfo, err := claims.ParseNamespace(grpcClientConfig.TokenNamespace)
 		if err != nil {
-			results[name] = contracts.NewDecryptResultErr(err)
-		} else {
-			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+			return nil, fmt.Errorf("failed to parse token namespace %v: %w", grpcClientConfig.TokenNamespace, err)
+		}
+		if nsInfo.OrgID < 1 {
+			return nil, fmt.Errorf("invalid token namepsace %v", grpcClientConfig.TokenNamespace)
+		}
+
+		tokenExchangeClient, err := authnlib.NewTokenExchangeClient(authnlib.TokenExchangeConfig{
+			Token:            grpcClientConfig.Token,
+			TokenExchangeURL: grpcClientConfig.TokenExchangeURL,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create token exchange client: %w", err)
+		}
+
+		tlsConfig := readTLSFromConfig(cfg)
+
+		client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, grpcClientConfig.TokenNamespace, cfg.SecretsManagement.DecryptServerAddress, tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create grpc decrypt client: %w", err)
+		}
+
+		return client, nil
+
+	case "local", "":
+		return NewLocalDecryptClient(decryptStorage)
+	}
+
+	return nil, fmt.Errorf("unsupported storage type: %s", cfg.SecretsManagement.DecryptServerType)
+}
+
+func readTLSFromConfig(cfg *setting.Cfg) TLSConfig {
+	if !cfg.SecretsManagement.DecryptServerUseTLS {
+		return TLSConfig{
+			UseTLS:             false,
+			InsecureSkipVerify: true,
 		}
 	}
 
-	return results, nil
+	apiServer := cfg.SectionWithEnvOverrides("grafana-apiserver")
+
+	return TLSConfig{
+		UseTLS:             true,
+		CertFile:           apiServer.Key("proxy_client_cert_file").MustString(""),
+		KeyFile:            apiServer.Key("proxy_client_key_file").MustString(""),
+		CAFile:             apiServer.Key("apiservice_ca_bundle_file").MustString(""),
+		ServerName:         cfg.SecretsManagement.DecryptServerTLSServerName,
+		InsecureSkipVerify: cfg.SecretsManagement.DecryptServerTLSSkipVerify,
+	}
 }

pkg/registry/apis/secret/decrypt/service_test.go

@@ -2,34 +2,52 @@ package decrypt
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 
+	decryptv1beta1 "github.com/grafana/grafana/apps/secret/decrypt/v1beta1"
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/madflojo/testcerts"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+	"golang.org/x/net/nettest"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/grafana/grafana/pkg/services/authn/clients"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 func TestDecryptService(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
+	tracer := noop.NewTracerProvider().Tracer("test")
 
 	t.Run("when there are only errors from the storage, the service returns them in the map", func(t *testing.T) {
 		t.Parallel()
 
 		mockErr := errors.New("mock error")
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		mockStorage.On("Decrypt", mock.Anything, mock.Anything, mock.Anything).Return(secretv1beta1.ExposedSecureValue(""), mockErr)
 		decryptedValuesResp := map[string]contracts.DecryptResult{
 			"secure-value-1": contracts.NewDecryptResultErr(mockErr),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1")
 		require.NotNil(t, resp)
@@ -40,7 +58,7 @@ func TestDecryptService(t *testing.T) {
 	t.Run("when there is no error from the storage, it returns a map of the decrypted values", func(t *testing.T) {
 		t.Parallel()
 
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		// Set up the mock to return a different value for each name in the test
 		exposedSecureValue1 := secretv1beta1.NewExposedSecureValue("value1")
 		exposedSecureValue2 := secretv1beta1.NewExposedSecureValue("value2")
@@ -54,9 +72,11 @@ func TestDecryptService(t *testing.T) {
 			"secure-value-2": contracts.NewDecryptResultValue(&exposedSecureValue2),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1", "secure-value-2")
 		require.NotNil(t, resp)
@@ -68,7 +88,7 @@ func TestDecryptService(t *testing.T) {
 		t.Parallel()
 
 		mockErr := errors.New("mock error")
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		exposedSecureValue := secretv1beta1.NewExposedSecureValue("value")
 		mockStorage.On("Decrypt", mock.Anything, xkube.Namespace("default"), "secure-value-1").
 			Return(exposedSecureValue, nil)
@@ -80,22 +100,213 @@ func TestDecryptService(t *testing.T) {
 			"secure-value-2": contracts.NewDecryptResultErr(mockErr),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1", "secure-value-2")
 		require.NotNil(t, resp)
 		require.NoError(t, err)
 		require.EqualValues(t, decryptedValuesResp, resp)
 	})
+
+	t.Run("when storage type is unsupported, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "unsupported"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "unsupported storage type")
+		require.Nil(t, decryptService)
+	})
+
+	t.Run("when storage type is grpc but token exchange config is missing, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+		cfg.SecretsManagement.DecryptServerAddress = "127.0.0.1:10000"
+
+		_, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "grpc_client_authentication.token and grpc_client_authentication.token_exchange_url are required")
+	})
+
+	t.Run("when storage type is grpc but storage address is missing, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+
+		_, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "decrypt_server_address is required")
+	})
+
+	t.Run("happy path with grpc+tls server with fake toke exchanger and server", func(t *testing.T) {
+		t.Parallel()
+
+		respTokenExchanged := "test-token"
+		tokenExchangeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			response := `{
+				"data": { 
+					"token": "` + respTokenExchanged + `"
+				}
+			}`
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(response))
+		}))
+		t.Cleanup(tokenExchangeServer.Close)
+
+		// Set up gRPC Server with TLS
+		listener, err := nettest.NewLocalListener("tcp")
+		require.NoError(t, err)
+
+		certPaths := createX509TestDir(t)
+
+		serverCert, err := tls.LoadX509KeyPair(certPaths.serverCert, certPaths.serverKey)
+		require.NoError(t, err)
+
+		tlsConfig := &tls.Config{
+			Certificates:       []tls.Certificate{serverCert},
+			ClientAuth:         tls.NoClientCert,
+			InsecureSkipVerify: false,
+			ServerName:         "localhost",
+		}
+
+		grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
+		t.Cleanup(grpcServer.Stop)
+
+		decryptServer := &mockDecryptServer{}
+		decryptServer.On("DecryptSecureValues", mock.Anything, mock.Anything).Return(
+			&decryptv1beta1.SecureValueDecryptResponseCollection{
+				DecryptedValues: map[string]*decryptv1beta1.Result{
+					"secure-value-1": {
+						Result: &decryptv1beta1.Result_Value{Value: "decrypted-value-1"},
+					},
+				},
+			},
+			nil,
+		)
+
+		decryptv1beta1.RegisterSecureValueDecrypterServer(grpcServer, decryptServer)
+
+		go func() {
+			_ = grpcServer.Serve(listener)
+			<-t.Context().Done()
+		}()
+
+		// Populate configuration with gRPC+TLS options and mock token exchanger
+		grafanaSvcIdentity := "svc-identity-decrypter"
+		namespace := "stacks-1234"
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+		cfg.SecretsManagement.DecryptServerAddress = listener.Addr().String()
+		cfg.SecretsManagement.DecryptGrafanaServiceName = grafanaSvcIdentity
+		cfg.SecretsManagement.DecryptServerUseTLS = true
+		cfg.SecretsManagement.DecryptServerTLSServerName = "localhost"
+		cfg.SecretsManagement.DecryptServerTLSSkipVerify = false
+
+		grpcClientAuth := cfg.Raw.Section("grpc_client_authentication")
+		_, err = grpcClientAuth.NewKey("token", "test-token")
+		require.NoError(t, err)
+		_, err = grpcClientAuth.NewKey("token_exchange_url", tokenExchangeServer.URL)
+		require.NoError(t, err)
+		_, err = grpcClientAuth.NewKey("token_namespace", namespace)
+		require.NoError(t, err)
+
+		apiServer := cfg.Raw.Section("grafana-apiserver")
+		_, err = apiServer.NewKey("proxy_client_cert_file", certPaths.clientCert)
+		require.NoError(t, err)
+		_, err = apiServer.NewKey("proxy_client_key_file", certPaths.clientKey)
+		require.NoError(t, err)
+		_, err = apiServer.NewKey("apiservice_ca_bundle_file", certPaths.ca)
+		require.NoError(t, err)
+
+		// Create and test decryption, using the mock grpc server as we dont test the business logic here
+		decryptService, err := ProvideDecryptService(cfg, tracer, nil)
+		require.NoError(t, err)
+		require.NotNil(t, decryptService)
+
+		t.Cleanup(func() { require.NoError(t, decryptService.Close()) })
+
+		svcIdentity := "provsysoning-test"
+		authCtx := identity.WithServiceIdentityContext(ctx, 1, identity.WithServiceIdentityName(svcIdentity))
+
+		result, err := decryptService.Decrypt(authCtx, namespace, "secure-value-1")
+		require.NoError(t, err)
+		require.NotNil(t, result)
+		require.Len(t, result, 1)
+		require.NotEmpty(t, result["secure-value-1"])
+
+		requestContext := decryptServer.Calls[0].Arguments[0].(context.Context)
+
+		md, ok := metadata.FromIncomingContext(requestContext)
+		require.True(t, ok)
+		require.NotEmpty(t, md)
+		require.Equal(t, svcIdentity, md[strings.ToLower(contracts.HeaderGrafanaServiceIdentityName)][0])
+		require.Equal(t, respTokenExchanged, md[strings.ToLower(clients.ExtJWTAuthenticationHeaderName)][0])
+	})
 }
 
-type MockDecryptStorage struct {
+type mockDecryptStorage struct {
 	mock.Mock
 }
 
-func (m *MockDecryptStorage) Decrypt(ctx context.Context, namespace xkube.Namespace, name string) (secretv1beta1.ExposedSecureValue, error) {
+func (m *mockDecryptStorage) Decrypt(ctx context.Context, namespace xkube.Namespace, name string) (secretv1beta1.ExposedSecureValue, error) {
 	args := m.Called(ctx, namespace, name)
 	return args.Get(0).(secretv1beta1.ExposedSecureValue), args.Error(1)
 }
+
+type mockDecryptServer struct {
+	mock.Mock
+}
+
+var _ decryptv1beta1.SecureValueDecrypterServer = (*mockDecryptServer)(nil)
+
+func (m *mockDecryptServer) DecryptSecureValues(ctx context.Context, req *decryptv1beta1.SecureValueDecryptRequest) (*decryptv1beta1.SecureValueDecryptResponseCollection, error) {
+	args := m.Called(ctx, req)
+	return args.Get(0).(*decryptv1beta1.SecureValueDecryptResponseCollection), args.Error(1)
+}
+
+type certPaths struct {
+	clientCert string
+	clientKey  string
+	serverCert string
+	serverKey  string
+	ca         string
+}
+
+func createX509TestDir(t *testing.T) certPaths {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+
+	ca := testcerts.NewCA()
+	caCertFile, _, err := ca.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	serverKp, err := ca.NewKeyPair("localhost")
+	require.NoError(t, err)
+
+	serverCertFile, serverKeyFile, err := serverKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	clientKp, err := ca.NewKeyPair()
+	require.NoError(t, err)
+	clientCertFile, clientKeyFile, err := clientKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	return certPaths{
+		clientCert: clientCertFile.Name(),
+		clientKey:  clientKeyFile.Name(),
+		serverCert: serverCertFile.Name(),
+		serverKey:  serverKeyFile.Name(),
+		ca:         caCertFile.Name(),
+	}
+}

pkg/registry/apis/secret/testutils/testutils.go

@@ -59,6 +59,7 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	}
 
 	tracer := noop.NewTracerProvider().Tracer("test")
+
 	testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
 
 	database := database.ProvideDatabase(testDB, tracer)
@@ -77,11 +78,11 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	})
 
 	defaultKey := "SdlklWklckeLS"
-	cfg := &setting.Cfg{
-		SecretsManagement: setting.SecretsManagerSettings{
-			CurrentEncryptionProvider: "secret_key.v1",
-			ConfiguredKMSProviders:    map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
-		},
+	cfg := setting.NewCfg()
+	cfg.SecretsManagement = setting.SecretsManagerSettings{
+		DecryptServerType:         "local",
+		CurrentEncryptionProvider: "secret_key.v1",
+		ConfiguredKMSProviders:    map[string]map[string]string{"secret_key.v1": {"secret_key": defaultKey}},
 	}
 	store, err := encryptionstorage.ProvideDataKeyStorage(database, tracer, nil)
 	require.NoError(t, err)
@@ -126,7 +127,13 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	decryptStorage, err := metadata.ProvideDecryptStorage(tracer, keeperService, keeperMetadataStorage, secureValueMetadataStorage, decryptAuthorizer, nil)
 	require.NoError(t, err)
 
-	decryptService := decrypt.ProvideDecryptService(decryptStorage)
+	testCfg := setting.NewCfg()
+	testCfg.SecretsManagement = setting.SecretsManagerSettings{
+		DecryptServerType: "local",
+	}
+
+	decryptService, err := decrypt.ProvideDecryptService(testCfg, tracer, decryptStorage)
+	require.NoError(t, err)
 
 	return Sut{
 		SecureValueService:          secureValueService,

pkg/server/wire_gen.go

@@ -788,8 +788,11 @@ func Initialize(cfg *setting.Cfg, opts Options, apiOpts api.ServerOptions) (*Ser
 	if err != nil {
 		return nil, err
 	}
-	v3 := decrypt.ProvideDecryptService(decryptStorage)
-	repositorySecrets := secrets.ProvideRepositorySecrets(featureToggles, secretsService, secureValueClient, v3)
+	v3, err := decrypt.ProvideDecryptService(cfg, tracer, decryptStorage)
+	if err != nil {
+		return nil, err
+	}
+	repositorySecrets := secrets.ProvideRepositorySecrets(featureToggles, secretsService, secureValueClient, v3, cfg)
 	webhookExtraBuilder := webhooks.ProvideWebhooks(cfg, featureToggles, repositorySecrets, factory, renderingService, resourceClient, eventualRestConfigProvider)
 	v4 := extras.ProvideProvisioningOSSExtras(webhookExtraBuilder)
 	apiBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, factory, accessClient, legacyMigrator, dualwriteService, usageStats, repositorySecrets, tracingService, v4)
@@ -1346,8 +1349,11 @@ func InitializeForTest(t sqlutil.ITestDB, testingT interface {
 	if err != nil {
 		return nil, err
 	}
-	v3 := decrypt.ProvideDecryptService(decryptStorage)
-	repositorySecrets := secrets.ProvideRepositorySecrets(featureToggles, secretsService, secureValueClient, v3)
+	v3, err := decrypt.ProvideDecryptService(cfg, tracer, decryptStorage)
+	if err != nil {
+		return nil, err
+	}
+	repositorySecrets := secrets.ProvideRepositorySecrets(featureToggles, secretsService, secureValueClient, v3, cfg)
 	webhookExtraBuilder := webhooks.ProvideWebhooks(cfg, featureToggles, repositorySecrets, factory, renderingService, resourceClient, eventualRestConfigProvider)
 	v4 := extras.ProvideProvisioningOSSExtras(webhookExtraBuilder)
 	apiBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, factory, accessClient, legacyMigrator, dualwriteService, usageStats, repositorySecrets, tracingService, v4)

pkg/setting/setting_secrets_manager.go

@@ -15,12 +15,26 @@ type SecretsManagerSettings struct {
 	// ConfiguredKMSProviders is a map of KMS providers found in the config file. The keys are in the format of <provider>.<keyName>, and the values are a map of the properties in that section
 	// In OSS, the provider type can only be "secret_key". In Enterprise, it can additionally be one of: "aws_kms", "azure_keyvault", "google_kms", "hashicorp_vault"
 	ConfiguredKMSProviders map[string]map[string]string
+
+	DecryptServerType          string // "local" or "grpc"
+	DecryptServerUseTLS        bool   // Applicable for decrypt_server_type=grpc. Whether to use TLS for the decrypt server
+	DecryptServerTLSSkipVerify bool   // Applicable for decrypt_server_type=grpc. Whether to skip TLS verification for the decrypt server
+	DecryptServerTLSServerName string // Applicable for decrypt_server_type=grpc. Server name to use for TLS verification
+	DecryptServerAddress       string // Applicable for decrypt_server_type=grpc. Address for external secrets server
+	DecryptGrafanaServiceName  string // Service name to use for background grafana decryption
 }
 
 func (cfg *Cfg) readSecretsManagerSettings() {
 	secretsMgmt := cfg.Raw.Section("secrets_manager")
 	cfg.SecretsManagement.CurrentEncryptionProvider = secretsMgmt.Key("encryption_provider").MustString(MisconfiguredProvider)
 
+	cfg.SecretsManagement.DecryptServerType = valueAsString(secretsMgmt, "decrypt_server_type", "local")
+	cfg.SecretsManagement.DecryptServerUseTLS = secretsMgmt.Key("decrypt_server_use_tls").MustBool(false)
+	cfg.SecretsManagement.DecryptServerTLSSkipVerify = secretsMgmt.Key("decrypt_server_tls_skip_verify").MustBool(false)
+	cfg.SecretsManagement.DecryptServerTLSServerName = valueAsString(secretsMgmt, "decrypt_server_tls_server_name", "")
+	cfg.SecretsManagement.DecryptServerAddress = valueAsString(secretsMgmt, "decrypt_server_address", "")
+	cfg.SecretsManagement.DecryptGrafanaServiceName = valueAsString(secretsMgmt, "decrypt_grafana_service_name", "")
+
 	// Extract available KMS providers from configuration sections
 	providers := make(map[string]map[string]string)
 	for _, section := range cfg.Raw.Sections() {

pkg/storage/secret/metadata/decrypt_store.go

@@ -7,12 +7,13 @@ import (
 	"time"
 
 	claims "github.com/grafana/authlib/types"
+	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/metadata"
 
-	"github.com/grafana/grafana-app-sdk/logging"
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
@@ -65,14 +66,33 @@ func (s *decryptStorage) Decrypt(ctx context.Context, namespace xkube.Namespace,
 	defer func() {
 		span.SetAttributes(attribute.String("decrypter.identity", decrypterIdentity))
 
+		args := []any{
+			"namespace", namespace.String(),
+			"secret_name", name,
+			"decrypter_identity", decrypterIdentity,
+		}
+
+		// The service identity used for decryption is always what is from the signed token, but if the request is
+		// coming from grafana, the service identity will be grafana, but the request metadata will contain
+		// additional service identity information (such as coming from the provisioning service in grafana).
+		// we do this for auditing purposes.
+		if md, ok := metadata.FromIncomingContext(ctx); ok {
+			if svcIdentities := md.Get(contracts.HeaderGrafanaServiceIdentityName); len(svcIdentities) > 0 {
+				args = append(args, "grafana_decrypter_identity", svcIdentities[0])
+				span.SetAttributes(attribute.String("grafana_decrypter.identity", svcIdentities[0]))
+			}
+		}
+
 		if decryptErr == nil {
-			logging.FromContext(ctx).Info("Audit log:", "operation", "decrypt_secret_success", "namespace", namespace, "secret_name", name, "decrypter_identity", decrypterIdentity)
+			args = append(args, "operation", "decrypt_secret_success")
 		} else {
 			span.SetStatus(codes.Error, "Decrypt failed")
 			span.RecordError(decryptErr)
-
-			logging.FromContext(ctx).Info("Audit log:", "operation", "decrypt_secret_error", "namespace", namespace, "secret_name", name, "decrypter_identity", decrypterIdentity, "error", decryptErr)
+			args = append(args, "operation", "decrypt_secret_error", "error", decryptErr.Error())
 		}
+
+		logging.FromContext(ctx).Info("Secrets Audit Log", args...)
+
 		success := decryptErr == nil
 		s.metrics.DecryptDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
 		s.metrics.DecryptRequestCount.WithLabelValues(strconv.FormatBool(success)).Inc()

pkg/storage/secret/metadata/decrypt_store_test.go

@@ -6,7 +6,9 @@ import (
 
 	"github.com/grafana/authlib/authn"
 	"github.com/grafana/authlib/types"
+	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/stretchr/testify/require"
+	grpcmetadata "google.golang.org/grpc/metadata"
 	"k8s.io/utils/ptr"
 
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
@@ -248,10 +250,67 @@ func TestIntegrationDecrypt(t *testing.T) {
 		require.Empty(t, exposed)
 	})
 
-	// TODO: add more tests for keeper failure scenarios, lets see how the async work will change this though.
+	t.Run("happy path with grpc metadata in request, also record the metadata as part of the service identity", func(t *testing.T) {
+		t.Parallel()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
+		tokenSvcIdentity := "svc"
+		stSvcIdentity := "st-svc"
+
+		// Create auth context with proper permissions that match the decrypters
+		authCtx := createAuthContext(ctx, "default", []string{"secret.grafana.app/securevalues:decrypt"}, tokenSvcIdentity, types.TypeUser)
+
+		// Needs to be incoming because we are pretending we received the metadata from a gRPC request
+		ctx = grpcmetadata.NewIncomingContext(authCtx, grpcmetadata.New(map[string]string{
+			contracts.HeaderGrafanaServiceIdentityName: stSvcIdentity,
+		}))
+
+		// Setup service
+		sut := testutils.Setup(t)
+
+		// Create a secure value
+		spec := secretv1beta1.SecureValueSpec{
+			Description: "description",
+			Decrypters:  []string{tokenSvcIdentity},
+			Value:       ptr.To(secretv1beta1.NewExposedSecureValue("value")),
+		}
+		sv := &secretv1beta1.SecureValue{Spec: spec}
+		sv.Name = "sv-test"
+		sv.Namespace = "default"
+
+		_, err := sut.CreateSv(ctx, testutils.CreateSvWithSv(sv))
+		require.NoError(t, err)
+
+		fakeLogger := &mockLogger{}
+
+		loggerCtx := logging.Context(ctx, fakeLogger)
+
+		exposed, err := sut.DecryptStorage.Decrypt(loggerCtx, "default", "sv-test")
+		require.NoError(t, err)
+		require.NotEmpty(t, exposed)
+		require.Equal(t, "value", exposed.DangerouslyExposeAndConsumeValue())
+
+		require.Len(t, fakeLogger.InfoArgs, 1)
+		args := fakeLogger.InfoArgs[0]
+		require.Contains(t, args, "grafana_decrypter_identity")
+		require.Contains(t, args, "decrypter_identity")
+		for i, arg := range args {
+			if arg == "grafana_decrypter_identity" {
+				require.Equal(t, stSvcIdentity, args[i+1].(string))
+			}
+
+			if arg == "decrypter_identity" {
+				require.Equal(t, tokenSvcIdentity, args[i+1].(string))
+			}
+		}
+	})
 }
 
 func createAuthContext(ctx context.Context, namespace string, permissions []string, svc string, identityType types.IdentityType) context.Context {
+	ctx = logging.Context(ctx, logging.DefaultLogger)
+
 	requester := &identity.StaticRequester{
 		Type:      identityType,
 		Namespace: namespace,
@@ -269,3 +328,18 @@ func createAuthContext(ctx context.Context, namespace string, permissions []stri
 
 	return types.WithAuthInfo(ctx, requester)
 }
+
+type mockLogger struct {
+	logging.Logger
+	InfoMsgs []string
+	InfoArgs [][]any
+}
+
+func (m *mockLogger) Info(msg string, args ...any) {
+	m.InfoMsgs = append(m.InfoMsgs, msg)
+	m.InfoArgs = append(m.InfoArgs, args)
+}
+
+func (m *mockLogger) WithContext(ctx context.Context) logging.Logger {
+	return m
+}