Secrets service: Allow decrypt through gRPC connection (#108365)

pkg/registry/apis/secret/decrypt/grpc_client.go

@@ -0,0 +1,178 @@
+package decrypt
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/fullstorydev/grpchan"
+	authnlib "github.com/grafana/authlib/authn"
+	claims "github.com/grafana/authlib/types"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+
+	decryptv1beta1 "github.com/grafana/grafana/apps/secret/decrypt/v1beta1"
+	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+)
+
+type GRPCDecryptClient struct {
+	conn   *grpc.ClientConn
+	client decryptv1beta1.SecureValueDecrypterClient
+}
+
+var _ contracts.DecryptService = &GRPCDecryptClient{}
+
+type TLSConfig struct {
+	UseTLS             bool
+	CertFile           string
+	KeyFile            string
+	CAFile             string
+	ServerName         string
+	InsecureSkipVerify bool
+}
+
+func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, namespace, address string) (*GRPCDecryptClient, error) {
+	return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, namespace, address, TLSConfig{})
+}
+
+func NewGRPCDecryptClientWithTLS(
+	tokenExchanger authnlib.TokenExchanger,
+	tracer trace.Tracer,
+	namespace string,
+	address string,
+	tlsConfig TLSConfig,
+) (*GRPCDecryptClient, error) {
+	var opts []grpc.DialOption
+	if tlsConfig.UseTLS {
+		creds, err := createTLSCredentials(tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to setup TLS: %w", err)
+		}
+
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+
+	conn, err := grpc.NewClient(address, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)
+	}
+
+	tokenExchangerInterceptor := authnlib.NewGrpcClientInterceptor(
+		tokenExchanger,
+		authnlib.WithClientInterceptorTracer(tracer),
+		authnlib.WithClientInterceptorNamespace(namespace),
+		authnlib.WithClientInterceptorAudience([]string{secretv1beta1.APIGroup}),
+	)
+
+	clientConn := grpchan.InterceptClientConn(
+		conn,
+		tokenExchangerInterceptor.UnaryClientInterceptor,
+		tokenExchangerInterceptor.StreamClientInterceptor,
+	)
+
+	return &GRPCDecryptClient{
+		conn:   conn,
+		client: decryptv1beta1.NewSecureValueDecrypterClient(clientConn),
+	}, nil
+}
+
+func createTLSCredentials(config TLSConfig) (credentials.TransportCredentials, error) {
+	tlsConfig := &tls.Config{}
+
+	if config.CAFile != "" {
+		caCert, err := os.ReadFile(config.CAFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA: %w", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to append CA")
+		}
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	if config.CertFile != "" && config.KeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load client certificate: %w", err)
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	if config.ServerName != "" {
+		tlsConfig.ServerName = config.ServerName
+	}
+
+	if config.InsecureSkipVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
+
+	return credentials.NewTLS(tlsConfig), nil
+}
+
+func (g *GRPCDecryptClient) Close() error {
+	if g.conn != nil {
+		return g.conn.Close()
+	}
+	return nil
+}
+
+func (g *GRPCDecryptClient) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
+	authInfo, ok := claims.AuthInfoFrom(ctx)
+	if !ok {
+		return nil, errors.New("missing auth info in context")
+	}
+
+	// Up until here the identity is the one set by the internal service, but when the request goes out to the gRPC server,
+	// the aggregator will use the access token which contains a different service identity for grafana as a whole.
+	// This is used for logging purposes only.
+	serviceIdentityList, ok := authInfo.GetExtra()[authnlib.ServiceIdentityKey]
+	if !ok || len(serviceIdentityList) != 1 {
+		return nil, errors.New("invalid service identity in auth info")
+	}
+
+	serviceIdentity := serviceIdentityList[0]
+	if len(serviceIdentity) == 0 {
+		return nil, errors.New("empty service identity in auth info")
+	}
+
+	req := &decryptv1beta1.SecureValueDecryptRequest{
+		Namespace: namespace,
+		Names:     names,
+	}
+
+	// Decryption will still use the service identity from the auth token,
+	// but we also pass the service identity from the request metadata for auditing purposes.
+	md := metadata.New(map[string]string{
+		contracts.HeaderGrafanaServiceIdentityName: serviceIdentity,
+	})
+	ctx = metadata.NewOutgoingContext(ctx, md)
+
+	resp, err := g.client.DecryptSecureValues(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("grpc decrypt failed: %w", err)
+	}
+
+	results := make(map[string]contracts.DecryptResult, len(resp.GetDecryptedValues()))
+
+	for name, result := range resp.GetDecryptedValues() {
+		if result.GetErrorMessage() != "" {
+			results[name] = contracts.NewDecryptResultErr(errors.New(result.GetErrorMessage()))
+		} else {
+			exposedSecureValue := secretv1beta1.NewExposedSecureValue(result.GetValue())
+			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+		}
+	}
+
+	return results, nil
+}

pkg/registry/apis/secret/decrypt/local_client.go

@@ -0,0 +1,39 @@
+package decrypt
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+)
+
+type LocalDecryptClient struct {
+	decryptStorage contracts.DecryptStorage
+}
+
+var _ contracts.DecryptService = &LocalDecryptClient{}
+
+func NewLocalDecryptClient(decryptStorage contracts.DecryptStorage) (*LocalDecryptClient, error) {
+	return &LocalDecryptClient{
+		decryptStorage: decryptStorage,
+	}, nil
+}
+
+func (c *LocalDecryptClient) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
+	results := make(map[string]contracts.DecryptResult, len(names))
+
+	for _, name := range names {
+		exposedSecureValue, err := c.decryptStorage.Decrypt(ctx, xkube.Namespace(namespace), name)
+		if err != nil {
+			results[name] = contracts.NewDecryptResultErr(err)
+		} else {
+			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+		}
+	}
+
+	return results, nil
+}
+
+func (c *LocalDecryptClient) Close() error {
+	return nil
+}

pkg/registry/apis/secret/decrypt/service.go

@@ -1,33 +1,78 @@
 package decrypt
 
 import (
-	"context"
+	"fmt"
+
+	authnlib "github.com/grafana/authlib/authn"
+	claims "github.com/grafana/authlib/types"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/grafana/grafana/pkg/services/authn/grpcutils"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
-type OSSDecryptService struct {
-	decryptStore contracts.DecryptStorage
-}
+func ProvideDecryptService(cfg *setting.Cfg, tracer trace.Tracer, decryptStorage contracts.DecryptStorage) (contracts.DecryptService, error) {
+	switch cfg.SecretsManagement.DecryptServerType {
+	case "grpc":
+		grpcClientConfig := grpcutils.ReadGrpcClientConfig(cfg)
 
-func ProvideDecryptService(decryptStore contracts.DecryptStorage) contracts.DecryptService {
-	return &OSSDecryptService{
-		decryptStore: decryptStore,
-	}
-}
+		if cfg.SecretsManagement.DecryptServerAddress == "" {
+			return nil, fmt.Errorf("decrypt_server_address is required when decrypt_server_type is grpc")
+		}
 
-func (d *OSSDecryptService) Decrypt(ctx context.Context, namespace string, names ...string) (map[string]contracts.DecryptResult, error) {
-	results := make(map[string]contracts.DecryptResult, len(names))
+		if grpcClientConfig.Token == "" || grpcClientConfig.TokenExchangeURL == "" {
+			return nil, fmt.Errorf("grpc_client_authentication.token and grpc_client_authentication.token_exchange_url are required when secrets_manager.decrypt_server_type is grpc")
+		}
 
-	for _, name := range names {
-		exposedSecureValue, err := d.decryptStore.Decrypt(ctx, xkube.Namespace(namespace), name)
+		nsInfo, err := claims.ParseNamespace(grpcClientConfig.TokenNamespace)
 		if err != nil {
-			results[name] = contracts.NewDecryptResultErr(err)
-		} else {
-			results[name] = contracts.NewDecryptResultValue(&exposedSecureValue)
+			return nil, fmt.Errorf("failed to parse token namespace %v: %w", grpcClientConfig.TokenNamespace, err)
+		}
+		if nsInfo.OrgID < 1 {
+			return nil, fmt.Errorf("invalid token namepsace %v", grpcClientConfig.TokenNamespace)
+		}
+
+		tokenExchangeClient, err := authnlib.NewTokenExchangeClient(authnlib.TokenExchangeConfig{
+			Token:            grpcClientConfig.Token,
+			TokenExchangeURL: grpcClientConfig.TokenExchangeURL,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create token exchange client: %w", err)
+		}
+
+		tlsConfig := readTLSFromConfig(cfg)
+
+		client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, grpcClientConfig.TokenNamespace, cfg.SecretsManagement.DecryptServerAddress, tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create grpc decrypt client: %w", err)
+		}
+
+		return client, nil
+
+	case "local", "":
+		return NewLocalDecryptClient(decryptStorage)
+	}
+
+	return nil, fmt.Errorf("unsupported storage type: %s", cfg.SecretsManagement.DecryptServerType)
+}
+
+func readTLSFromConfig(cfg *setting.Cfg) TLSConfig {
+	if !cfg.SecretsManagement.DecryptServerUseTLS {
+		return TLSConfig{
+			UseTLS:             false,
+			InsecureSkipVerify: true,
 		}
 	}
 
-	return results, nil
+	apiServer := cfg.SectionWithEnvOverrides("grafana-apiserver")
+
+	return TLSConfig{
+		UseTLS:             true,
+		CertFile:           apiServer.Key("proxy_client_cert_file").MustString(""),
+		KeyFile:            apiServer.Key("proxy_client_key_file").MustString(""),
+		CAFile:             apiServer.Key("apiservice_ca_bundle_file").MustString(""),
+		ServerName:         cfg.SecretsManagement.DecryptServerTLSServerName,
+		InsecureSkipVerify: cfg.SecretsManagement.DecryptServerTLSSkipVerify,
+	}
 }

pkg/registry/apis/secret/decrypt/service_test.go

@@ -2,34 +2,52 @@ package decrypt
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 
+	decryptv1beta1 "github.com/grafana/grafana/apps/secret/decrypt/v1beta1"
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
-	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/madflojo/testcerts"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+	"golang.org/x/net/nettest"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
+	"github.com/grafana/grafana/pkg/services/authn/clients"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 func TestDecryptService(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
+	tracer := noop.NewTracerProvider().Tracer("test")
 
 	t.Run("when there are only errors from the storage, the service returns them in the map", func(t *testing.T) {
 		t.Parallel()
 
 		mockErr := errors.New("mock error")
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		mockStorage.On("Decrypt", mock.Anything, mock.Anything, mock.Anything).Return(secretv1beta1.ExposedSecureValue(""), mockErr)
 		decryptedValuesResp := map[string]contracts.DecryptResult{
 			"secure-value-1": contracts.NewDecryptResultErr(mockErr),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1")
 		require.NotNil(t, resp)
@@ -40,7 +58,7 @@ func TestDecryptService(t *testing.T) {
 	t.Run("when there is no error from the storage, it returns a map of the decrypted values", func(t *testing.T) {
 		t.Parallel()
 
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		// Set up the mock to return a different value for each name in the test
 		exposedSecureValue1 := secretv1beta1.NewExposedSecureValue("value1")
 		exposedSecureValue2 := secretv1beta1.NewExposedSecureValue("value2")
@@ -54,9 +72,11 @@ func TestDecryptService(t *testing.T) {
 			"secure-value-2": contracts.NewDecryptResultValue(&exposedSecureValue2),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1", "secure-value-2")
 		require.NotNil(t, resp)
@@ -68,7 +88,7 @@ func TestDecryptService(t *testing.T) {
 		t.Parallel()
 
 		mockErr := errors.New("mock error")
-		mockStorage := &MockDecryptStorage{}
+		mockStorage := &mockDecryptStorage{}
 		exposedSecureValue := secretv1beta1.NewExposedSecureValue("value")
 		mockStorage.On("Decrypt", mock.Anything, xkube.Namespace("default"), "secure-value-1").
 			Return(exposedSecureValue, nil)
@@ -80,22 +100,213 @@ func TestDecryptService(t *testing.T) {
 			"secure-value-2": contracts.NewDecryptResultErr(mockErr),
 		}
 
-		decryptService := &OSSDecryptService{
-			decryptStore: mockStorage,
-		}
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "local"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, mockStorage)
+		require.NoError(t, err)
 
 		resp, err := decryptService.Decrypt(ctx, "default", "secure-value-1", "secure-value-2")
 		require.NotNil(t, resp)
 		require.NoError(t, err)
 		require.EqualValues(t, decryptedValuesResp, resp)
 	})
+
+	t.Run("when storage type is unsupported, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "unsupported"
+
+		decryptService, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "unsupported storage type")
+		require.Nil(t, decryptService)
+	})
+
+	t.Run("when storage type is grpc but token exchange config is missing, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+		cfg.SecretsManagement.DecryptServerAddress = "127.0.0.1:10000"
+
+		_, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "grpc_client_authentication.token and grpc_client_authentication.token_exchange_url are required")
+	})
+
+	t.Run("when storage type is grpc but storage address is missing, it returns an error", func(t *testing.T) {
+		t.Parallel()
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+
+		_, err := ProvideDecryptService(cfg, tracer, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "decrypt_server_address is required")
+	})
+
+	t.Run("happy path with grpc+tls server with fake toke exchanger and server", func(t *testing.T) {
+		t.Parallel()
+
+		respTokenExchanged := "test-token"
+		tokenExchangeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			response := `{
+				"data": { 
+					"token": "` + respTokenExchanged + `"
+				}
+			}`
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(response))
+		}))
+		t.Cleanup(tokenExchangeServer.Close)
+
+		// Set up gRPC Server with TLS
+		listener, err := nettest.NewLocalListener("tcp")
+		require.NoError(t, err)
+
+		certPaths := createX509TestDir(t)
+
+		serverCert, err := tls.LoadX509KeyPair(certPaths.serverCert, certPaths.serverKey)
+		require.NoError(t, err)
+
+		tlsConfig := &tls.Config{
+			Certificates:       []tls.Certificate{serverCert},
+			ClientAuth:         tls.NoClientCert,
+			InsecureSkipVerify: false,
+			ServerName:         "localhost",
+		}
+
+		grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
+		t.Cleanup(grpcServer.Stop)
+
+		decryptServer := &mockDecryptServer{}
+		decryptServer.On("DecryptSecureValues", mock.Anything, mock.Anything).Return(
+			&decryptv1beta1.SecureValueDecryptResponseCollection{
+				DecryptedValues: map[string]*decryptv1beta1.Result{
+					"secure-value-1": {
+						Result: &decryptv1beta1.Result_Value{Value: "decrypted-value-1"},
+					},
+				},
+			},
+			nil,
+		)
+
+		decryptv1beta1.RegisterSecureValueDecrypterServer(grpcServer, decryptServer)
+
+		go func() {
+			_ = grpcServer.Serve(listener)
+			<-t.Context().Done()
+		}()
+
+		// Populate configuration with gRPC+TLS options and mock token exchanger
+		grafanaSvcIdentity := "svc-identity-decrypter"
+		namespace := "stacks-1234"
+
+		cfg := setting.NewCfg()
+		cfg.SecretsManagement.DecryptServerType = "grpc"
+		cfg.SecretsManagement.DecryptServerAddress = listener.Addr().String()
+		cfg.SecretsManagement.DecryptGrafanaServiceName = grafanaSvcIdentity
+		cfg.SecretsManagement.DecryptServerUseTLS = true
+		cfg.SecretsManagement.DecryptServerTLSServerName = "localhost"
+		cfg.SecretsManagement.DecryptServerTLSSkipVerify = false
+
+		grpcClientAuth := cfg.Raw.Section("grpc_client_authentication")
+		_, err = grpcClientAuth.NewKey("token", "test-token")
+		require.NoError(t, err)
+		_, err = grpcClientAuth.NewKey("token_exchange_url", tokenExchangeServer.URL)
+		require.NoError(t, err)
+		_, err = grpcClientAuth.NewKey("token_namespace", namespace)
+		require.NoError(t, err)
+
+		apiServer := cfg.Raw.Section("grafana-apiserver")
+		_, err = apiServer.NewKey("proxy_client_cert_file", certPaths.clientCert)
+		require.NoError(t, err)
+		_, err = apiServer.NewKey("proxy_client_key_file", certPaths.clientKey)
+		require.NoError(t, err)
+		_, err = apiServer.NewKey("apiservice_ca_bundle_file", certPaths.ca)
+		require.NoError(t, err)
+
+		// Create and test decryption, using the mock grpc server as we dont test the business logic here
+		decryptService, err := ProvideDecryptService(cfg, tracer, nil)
+		require.NoError(t, err)
+		require.NotNil(t, decryptService)
+
+		t.Cleanup(func() { require.NoError(t, decryptService.Close()) })
+
+		svcIdentity := "provsysoning-test"
+		authCtx := identity.WithServiceIdentityContext(ctx, 1, identity.WithServiceIdentityName(svcIdentity))
+
+		result, err := decryptService.Decrypt(authCtx, namespace, "secure-value-1")
+		require.NoError(t, err)
+		require.NotNil(t, result)
+		require.Len(t, result, 1)
+		require.NotEmpty(t, result["secure-value-1"])
+
+		requestContext := decryptServer.Calls[0].Arguments[0].(context.Context)
+
+		md, ok := metadata.FromIncomingContext(requestContext)
+		require.True(t, ok)
+		require.NotEmpty(t, md)
+		require.Equal(t, svcIdentity, md[strings.ToLower(contracts.HeaderGrafanaServiceIdentityName)][0])
+		require.Equal(t, respTokenExchanged, md[strings.ToLower(clients.ExtJWTAuthenticationHeaderName)][0])
+	})
 }
 
-type MockDecryptStorage struct {
+type mockDecryptStorage struct {
 	mock.Mock
 }
 
-func (m *MockDecryptStorage) Decrypt(ctx context.Context, namespace xkube.Namespace, name string) (secretv1beta1.ExposedSecureValue, error) {
+func (m *mockDecryptStorage) Decrypt(ctx context.Context, namespace xkube.Namespace, name string) (secretv1beta1.ExposedSecureValue, error) {
 	args := m.Called(ctx, namespace, name)
 	return args.Get(0).(secretv1beta1.ExposedSecureValue), args.Error(1)
 }
+
+type mockDecryptServer struct {
+	mock.Mock
+}
+
+var _ decryptv1beta1.SecureValueDecrypterServer = (*mockDecryptServer)(nil)
+
+func (m *mockDecryptServer) DecryptSecureValues(ctx context.Context, req *decryptv1beta1.SecureValueDecryptRequest) (*decryptv1beta1.SecureValueDecryptResponseCollection, error) {
+	args := m.Called(ctx, req)
+	return args.Get(0).(*decryptv1beta1.SecureValueDecryptResponseCollection), args.Error(1)
+}
+
+type certPaths struct {
+	clientCert string
+	clientKey  string
+	serverCert string
+	serverKey  string
+	ca         string
+}
+
+func createX509TestDir(t *testing.T) certPaths {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+
+	ca := testcerts.NewCA()
+	caCertFile, _, err := ca.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	serverKp, err := ca.NewKeyPair("localhost")
+	require.NoError(t, err)
+
+	serverCertFile, serverKeyFile, err := serverKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	clientKp, err := ca.NewKeyPair()
+	require.NoError(t, err)
+	clientCertFile, clientKeyFile, err := clientKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	return certPaths{
+		clientCert: clientCertFile.Name(),
+		clientKey:  clientKeyFile.Name(),
+		serverCert: serverCertFile.Name(),
+		serverKey:  serverKeyFile.Name(),
+		ca:         caCertFile.Name(),
+	}
+}