From d0792ebe9715ed91851c6eaf5a6494485cfe37ef Mon Sep 17 00:00:00 2001 From: Charandas <542168+charandas@users.noreply.github.com> Date: Thu, 18 Dec 2025 07:44:33 -0800 Subject: [PATCH] Secrets: Add gRPC client retry with exp. backoff" (#115526) Provisioning: secrets decrypt client should retry with exponential backoff --- pkg/registry/apis/secret/decrypt/grpc_client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/registry/apis/secret/decrypt/grpc_client.go b/pkg/registry/apis/secret/decrypt/grpc_client.go index f892de5ebe5..0612455c3d8 100644 --- a/pkg/registry/apis/secret/decrypt/grpc_client.go +++ b/pkg/registry/apis/secret/decrypt/grpc_client.go @@ -9,10 +9,13 @@ import ( "maps" "os" "slices" + "time" "github.com/fullstorydev/grpchan" + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" @@ -72,6 +75,15 @@ func NewGRPCDecryptClientWithTLS( opts = append(opts, grpc.WithDisableServiceConfig()) } + // Add retry interceptor to retry on transient connection issues. + // Retries on ResourceExhausted (per-RPC limits reached) and Unavailable (system unavailable). + retryInterceptor := grpc_retry.UnaryClientInterceptor( + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(time.Second, 0.5)), + grpc_retry.WithCodes(codes.ResourceExhausted, codes.Unavailable), + ) + opts = append(opts, grpc.WithUnaryInterceptor(retryInterceptor)) + conn, err := grpc.NewClient(address, opts...) if err != nil { return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)