Secrets: Add gRPC client retry with exp. backoff" (#115526)

Provisioning: secrets decrypt client should retry with exponential backoff

pkg/registry/apis/secret/decrypt/grpc_client.go

@@ -9,10 +9,13 @@ import (
 	"maps"
 	"os"
 	"slices"
+	"time"
 
 	"github.com/fullstorydev/grpchan"
+	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/metadata"
@@ -72,6 +75,15 @@ func NewGRPCDecryptClientWithTLS(
 		opts = append(opts, grpc.WithDisableServiceConfig())
 	}
 
+	// Add retry interceptor to retry on transient connection issues.
+	// Retries on ResourceExhausted (per-RPC limits reached) and Unavailable (system unavailable).
+	retryInterceptor := grpc_retry.UnaryClientInterceptor(
+		grpc_retry.WithMax(3),
+		grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(time.Second, 0.5)),
+		grpc_retry.WithCodes(codes.ResourceExhausted, codes.Unavailable),
+	)
+	opts = append(opts, grpc.WithUnaryInterceptor(retryInterceptor))
+
 	conn, err := grpc.NewClient(address, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)