grafana/pkg/registry/apis/provisioning/resources/retry_client.go

package resources

import (
	"context"
	"errors"
	"net"
	"time"

	"github.com/grafana/grafana-app-sdk/logging"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/dynamic"
)

// defaultRetryBackoff returns a default backoff configuration for retries.
//
// Retry attempts will happen when:
//   - The Kubernetes API returns transient errors: ServiceUnavailable (503), ServerTimeout (504),
//     TooManyRequests (429), InternalError (500), or Timeout errors
//   - Network errors occur: connection timeouts, temporary network failures, or connection errors
//
// The retry behavior:
//   - Total attempts: 8 (1 initial attempt + 7 retries)
//   - Initial delay: 100ms before the first retry
//   - Exponential backoff: delay doubles after each failed attempt (100ms → 200ms → 400ms → 800ms → 1.6s → 3.2s → 5s)
//   - Maximum delay: capped at 5 seconds
//   - Jitter: 10% randomization to prevent thundering herd problems
//   - Total retry window: approximately 10 seconds from first attempt to last retry
//
// All attempts will fail when:
//   - The Kubernetes API server is completely unavailable or unreachable
//   - Network connectivity issues persist beyond the retry window (~10 seconds)
//   - The API server returns transient errors consistently for the entire retry duration
//   - Context cancellation occurs before retries complete
//
// Non-transient errors (e.g., NotFound, BadRequest, Forbidden) are not retried and returned immediately.
func defaultRetryBackoff() wait.Backoff {
	return wait.Backoff{
		Duration: 100 * time.Millisecond,
		Factor:   2.0,
		Jitter:   0.1,
		Steps:    8, // 1 initial attempt + 7 retries = 8 total attempts (~10s total retry window)
		Cap:      5 * time.Second,
	}
}

// retryResourceInterface wraps a dynamic.ResourceInterface with retry logic for transient errors
type retryResourceInterface struct {
	client  dynamic.ResourceInterface
	backoff wait.Backoff
}

// newRetryResourceInterface creates a new ResourceInterface wrapper with retry logic
func newRetryResourceInterface(client dynamic.ResourceInterface, backoff wait.Backoff) dynamic.ResourceInterface {
	return &retryResourceInterface{
		client:  client,
		backoff: backoff,
	}
}

// isTransientError determines if an error is transient and should be retried
func isTransientError(err error) bool {
	if err == nil {
		return false
	}

	// Check for Kubernetes API transient errors
	if apierrors.IsServiceUnavailable(err) {
		return true
	}
	if apierrors.IsServerTimeout(err) {
		return true
	}
	if apierrors.IsTooManyRequests(err) {
		return true
	}
	if apierrors.IsInternalError(err) {
		return true
	}
	if apierrors.IsTimeout(err) {
		return true
	}

	// Check for network errors
	var netErr net.Error
	if errors.As(err, &netErr) {
		if netErr.Timeout() {
			return true
		}
	}

	// Check for connection errors
	var opErr *net.OpError
	return errors.As(err, &opErr)
}

// retryWithBackoff executes a function with exponential backoff retry logic using wait.ExponentialBackoff
func (r *retryResourceInterface) retryWithBackoff(ctx context.Context, fn func() error) error {
	var lastErr error
	attempt := 0
	logger := logging.FromContext(ctx)

	err := wait.ExponentialBackoff(r.backoff, func() (bool, error) {
		attempt++

		// Check if context is cancelled
		if ctx.Err() != nil {
			logger.Debug("Retry cancelled due to context cancellation", "attempt", attempt)
			return false, ctx.Err()
		}

		err := fn()
		if err == nil {
			if attempt > 1 {
				logger.Debug("Operation succeeded after retry", "attempt", attempt)
			}
			return true, nil // success, stop retrying
		}

		// If not a transient error, return immediately without retrying
		if !isTransientError(err) {
			logger.Debug("Non-transient error, not retrying", "attempt", attempt, "error", err)
			return false, err
		}

		// Transient error, retry
		lastErr = err
		logger.Info("Transient error encountered, retrying", "attempt", attempt, "max_attempts", r.backoff.Steps, "error", err)
		return false, nil
	})

	// If wait.ExponentialBackoff returned an error, it means we exhausted retries
	if err != nil {
		if lastErr != nil {
			logger.Warn("All retry attempts exhausted", "total_attempts", attempt, "error", lastErr)
			return lastErr
		}
		logger.Warn("All retry attempts exhausted", "total_attempts", attempt, "error", err)
		return err
	}

	return nil
}

// Create implements dynamic.ResourceInterface
func (r *retryResourceInterface) Create(ctx context.Context, obj *unstructured.Unstructured, options metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.Create(ctx, obj, options, subresources...)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// Update implements dynamic.ResourceInterface
func (r *retryResourceInterface) Update(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.Update(ctx, obj, options, subresources...)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// UpdateStatus implements dynamic.ResourceInterface
func (r *retryResourceInterface) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.UpdateStatus(ctx, obj, options)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// Delete implements dynamic.ResourceInterface
func (r *retryResourceInterface) Delete(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error {
	return r.retryWithBackoff(ctx, func() error {
		return r.client.Delete(ctx, name, options, subresources...)
	})
}

// DeleteCollection implements dynamic.ResourceInterface
func (r *retryResourceInterface) DeleteCollection(ctx context.Context, options metav1.DeleteOptions, listOptions metav1.ListOptions) error {
	return r.retryWithBackoff(ctx, func() error {
		return r.client.DeleteCollection(ctx, options, listOptions)
	})
}

// Get implements dynamic.ResourceInterface
func (r *retryResourceInterface) Get(ctx context.Context, name string, options metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.Get(ctx, name, options, subresources...)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// List implements dynamic.ResourceInterface
func (r *retryResourceInterface) List(ctx context.Context, opts metav1.ListOptions) (*unstructured.UnstructuredList, error) {
	var result *unstructured.UnstructuredList
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.List(ctx, opts)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// Watch implements dynamic.ResourceInterface
func (r *retryResourceInterface) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
	// Watch operations are long-lived and shouldn't be retried in the same way
	// Return the watch interface directly
	return r.client.Watch(ctx, opts)
}

// Patch implements dynamic.ResourceInterface
func (r *retryResourceInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.Patch(ctx, name, pt, data, options, subresources...)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// Apply implements dynamic.ResourceInterface
func (r *retryResourceInterface) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.Apply(ctx, name, obj, options, subresources...)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}

// ApplyStatus implements dynamic.ResourceInterface
func (r *retryResourceInterface) ApplyStatus(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions) (*unstructured.Unstructured, error) {
	var result *unstructured.Unstructured
	var err error

	retryErr := r.retryWithBackoff(ctx, func() error {
		result, err = r.client.ApplyStatus(ctx, name, obj, options)
		return err
	})

	if retryErr != nil {
		return nil, retryErr
	}
	return result, nil
}