diff --git a/pkg/registry/apis/secret/decrypt/grpc_client.go b/pkg/registry/apis/secret/decrypt/grpc_client.go index f892de5ebe5..0612455c3d8 100644 --- a/pkg/registry/apis/secret/decrypt/grpc_client.go +++ b/pkg/registry/apis/secret/decrypt/grpc_client.go @@ -9,10 +9,13 @@ import ( "maps" "os" "slices" + "time" "github.com/fullstorydev/grpchan" + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" @@ -72,6 +75,15 @@ func NewGRPCDecryptClientWithTLS( opts = append(opts, grpc.WithDisableServiceConfig()) } + // Add retry interceptor to retry on transient connection issues. + // Retries on ResourceExhausted (per-RPC limits reached) and Unavailable (system unavailable). + retryInterceptor := grpc_retry.UnaryClientInterceptor( + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(time.Second, 0.5)), + grpc_retry.WithCodes(codes.ResourceExhausted, codes.Unavailable), + ) + opts = append(opts, grpc.WithUnaryInterceptor(retryInterceptor)) + conn, err := grpc.NewClient(address, opts...) if err != nil { return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)