fix: improve resilience for unified storage and search service grpc clients (#116122)

* fix: reliability

* fix: resilience

* fix: add connection backoff

* fix: reduce backoff

pkg/storage/unified/client.go

@@ -271,7 +271,7 @@ func grpcConn(address string, metrics *clientMetrics, clientKeepaliveTime time.D
 	retryCfg := retryConfig{
 		Max:           3,
 		Backoff:       time.Second,
-		BackoffJitter: 0.5,
+		BackoffJitter: 0.1,
 	}
 	unary = append(unary, unaryRetryInterceptor(retryCfg))
 	unary = append(unary, unaryRetryInstrument(metrics.requestRetries))
@@ -288,13 +288,15 @@ func grpcConn(address string, metrics *clientMetrics, clientKeepaliveTime time.D
 	opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
 	opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 
-	// Use round_robin to balances requests more evenly over the available Storage server.
+	// Use round_robin to balance requests more evenly over the available Storage server.
 	opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`))
 
 	// Disable looking up service config from TXT DNS records.
 	// This reduces the number of requests made to the DNS servers.
 	opts = append(opts, grpc.WithDisableServiceConfig())
 
+	opts = append(opts, connectionBackoffOptions())
+
 	if clientKeepaliveTime > 0 {
 		opts = append(opts, grpc.WithKeepaliveParams(keepalive.ClientParameters{
 			Time:                clientKeepaliveTime,

pkg/storage/unified/client_retry.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
 	"github.com/prometheus/client_golang/prometheus"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/codes"
 )
 
@@ -44,3 +45,17 @@ func unaryRetryInstrument(metric *prometheus.CounterVec) grpc.UnaryClientInterce
 		return invoker(ctx, method, req, resp, cc, opts...)
 	}
 }
+
+// connectionBackoffOptions configures connection backoff parameters for faster recovery from
+// transient connection failures (e.g., during pod restarts).
+func connectionBackoffOptions() grpc.DialOption {
+	return grpc.WithConnectParams(grpc.ConnectParams{
+		Backoff: backoff.Config{
+			BaseDelay:  100 * time.Millisecond,
+			Multiplier: 1.6,
+			Jitter:     0.2,
+			MaxDelay:   10 * time.Second,
+		},
+		MinConnectTimeout: 5 * time.Second,
+	})
+}