feat(unified-storage): add some basic retry logic for the grpc client (#101001)

pkg/storage/unified/client_retry.go

@@ -0,0 +1,46 @@
+package unified
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
+	"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
+	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+type retryConfig struct {
+	Max           uint
+	Backoff       time.Duration
+	BackoffJitter float64
+}
+
+// unaryRetryInterceptor creates an interceptor to perform retries for unary methods.
+//
+// Note: Retry codes are the same as the default codes.
+//
+//	From go-grpc-middleware/interceptors/retry/options.go:
+//	`ResourceExhausted` means that the user quota, e.g. per-RPC limits, have been reached.
+//	`Unavailable` means that system is currently unavailable and the client should retry again.
+func unaryRetryInterceptor(cfg retryConfig) grpc.UnaryClientInterceptor {
+	return grpc_retry.UnaryClientInterceptor(
+		grpc_retry.WithMax(cfg.Max),
+		grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(cfg.Backoff, cfg.BackoffJitter)),
+		grpc_retry.WithCodes(codes.ResourceExhausted, codes.Unavailable),
+	)
+}
+
+// unaryRetryInstrument creates an interceptor to count and log retry attempts.
+func unaryRetryInstrument(metric *prometheus.CounterVec) grpc.UnaryClientInterceptor {
+	return func(ctx context.Context, method string, req, resp interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		// We can tell if a call is a retry by checking the retry attempt metadata.
+		attempt, err := strconv.Atoi(metautils.ExtractOutgoing(ctx).Get(grpc_retry.AttemptMetadataKey))
+		if err == nil && attempt > 0 {
+			metric.WithLabelValues(method).Inc()
+		}
+		return invoker(ctx, method, req, resp, cc, opts...)
+	}
+}