Advisor: Retry before patching objects (#111665)

apps/advisor/pkg/app/utils.go

@@ -14,6 +14,7 @@ import (
 	advisorv0alpha1 "github.com/grafana/grafana/apps/advisor/pkg/apis/advisor/v0alpha1"
 	"github.com/grafana/grafana/apps/advisor/pkg/app/checks"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 var retryAnnotationPollingInterval = 1 * time.Second
@@ -81,7 +82,11 @@ func processCheck(ctx context.Context, log logging.Logger, client resource.Clien
 		}
 		return fmt.Errorf("error running steps: %w", err)
 	}
-
+	// Wait for the item to be persisted before patching the object
+	err = waitForItem(ctx, log, client, obj)
+	if err != nil {
+		return err
+	}
 	report := &advisorv0alpha1.CheckReport{
 		Failures: failures,
 		Count:    int64(len(items)),
@@ -264,6 +269,24 @@ func retryAnnotationChanged(oldObj, newObj resource.Object) bool {
 		oldAnnotations[checks.RetryAnnotation] != newAnnotations[checks.RetryAnnotation]
 }
 
+func waitForItem(ctx context.Context, log logging.Logger, client resource.Client, obj resource.Object) error {
+	_, err := client.Get(ctx, resource.Identifier{
+		Namespace: obj.GetNamespace(),
+		Name:      obj.GetName(),
+	})
+	retries := 0
+	for err != nil && k8serrors.IsNotFound(err) && retries < 5 {
+		log.Debug("Waiting for item to be persisted", "check", obj.GetName(), "retries", retries)
+		time.Sleep(retryAnnotationPollingInterval)
+		retries++
+		_, err = client.Get(ctx, resource.Identifier{
+			Namespace: obj.GetNamespace(),
+			Name:      obj.GetName(),
+		})
+	}
+	return err
+}
+
 // waitForRetryAnnotation waits for the retry annotation to match the item to retry
 func waitForRetryAnnotation(ctx context.Context, log logging.Logger, client resource.Client, obj resource.Object, itemToRetry string) error {
 	currentObj, err := client.Get(ctx, resource.Identifier{