Provisioning: Fix flaky tests and race condition in folder existing check (#111209)

Provisioning: Fix some of the flakiness

pkg/registry/apis/provisioning/resources/folders.go

@@ -15,6 +15,7 @@ import (
 	"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"github.com/grafana/grafana/pkg/services/dashboards"
 )
 
 const MaxNumberOfFolders = 10000
@@ -141,6 +142,28 @@ func (fm *FolderManager) EnsureFolderExists(ctx context.Context, folder Folder,
 	})
 
 	if _, err := fm.client.Create(ctx, obj, metav1.CreateOptions{}); err != nil {
+		// there is a potential race here where two syncs can be triggered
+		// if we try to create and there is an error, check if it is from another sync
+		// job for this repo that created it
+		if apierrors.IsAlreadyExists(err) || err.Error() == dashboards.ErrFolderVersionMismatch.Error() {
+			obj, err2 := fm.client.Get(ctx, folder.ID, metav1.GetOptions{})
+			if err2 != nil {
+				return fmt.Errorf("failed to get folder: %w", err2)
+			} else if obj == nil {
+				return fmt.Errorf("failed to create folder: %w", err)
+			}
+
+			current, ok := obj.GetAnnotations()[utils.AnnoKeyManagerIdentity]
+			if !ok {
+				return fmt.Errorf("target folder is not managed by a repository")
+			}
+			if current != cfg.Name {
+				return fmt.Errorf("target folder is managed by a different repository (%s)", current)
+			}
+
+			return nil
+		}
+
 		return fmt.Errorf("failed to create folder: %w", err)
 	}
 	return nil