feat(unified-storage): add some basic retry logic for the grpc client (#101001)

pkg/storage/unified/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	otgrpc "github.com/opentracing-contrib/go-grpc"
 	"github.com/opentracing/opentracing-go"
@@ -47,6 +48,7 @@ type Options struct {
 
 type clientMetrics struct {
 	requestDuration *prometheus.HistogramVec
+	requestRetries  *prometheus.CounterVec
 }
 
 // This adds a UnifiedStorage client into the wire dependency tree
@@ -171,11 +173,25 @@ func GrpcConn(address string, reg prometheus.Registerer) (*grpc.ClientConn, erro
 			Help:    "Time spent executing requests to the resource server.",
 			Buckets: prometheus.ExponentialBuckets(0.008, 4, 7),
 		}, []string{"operation", "status_code"}),
+		requestRetries: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+			Name: "resource_server_client_request_retries_total",
+			Help: "Total number of retries for requests to the resource server.",
+		}, []string{"operation"}),
 	}
 
 	// Report gRPC status code errors as labels.
 	unary, stream := instrument(metrics.requestDuration, middleware.ReportGRPCStatusOption)
 
+	// Add middleware to retry on transient connection issues. Note that
+	// we do not implement it for streams, as we don't currently use streams.
+	retryCfg := retryConfig{
+		Max:           3,
+		Backoff:       time.Second,
+		BackoffJitter: 0.5,
+	}
+	unary = append(unary, unaryRetryInterceptor(retryCfg))
+	unary = append(unary, unaryRetryInstrument(metrics.requestRetries))
+
 	cfg := grpcclient.Config{}
 	// Set the defaults that are normally set by Config.RegisterFlags.
 	flagext.DefaultValues(&cfg)