Provisioning: Fix flaky tests and race condition in folder existing check (#111209)

Provisioning: Fix some of the flakiness

pkg/tests/apis/provisioning/pulljob_test.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -19,9 +21,6 @@ import (
 )
 
 func TestIntegrationProvisioning_PullJobOwnershipProtection(t *testing.T) {
-	// TODO: fix flaky test
-	t.Skip("skipping flaky test")
-
 	testutil.SkipIntegrationTestInShortMode(t)
 
 	helper := runGrafana(t)
@@ -31,29 +30,58 @@ func TestIntegrationProvisioning_PullJobOwnershipProtection(t *testing.T) {
 	const repo1 = "pulljob-repo-1"
 	const repo2 = "pulljob-repo-2"
 
-	// Create first repository targeting "folder" with its own subdirectory
-	helper.CreateRepo(t, TestRepo{
-		Name:   repo1,
-		Path:   path.Join(helper.ProvisioningPath, "repo1"),
-		Target: "folder",
-		Copies: map[string]string{
-			"testdata/all-panels.json": "dashboard1.json",
-		},
-		ExpectedDashboards: 1,
-		ExpectedFolders:    1,
-	})
+	// create both repos concurrently to reduce duration of this test
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		helper.CreateRepo(t, TestRepo{
+			Name:   repo1,
+			Path:   path.Join(helper.ProvisioningPath, "repo1"),
+			Target: "folder",
+			Copies: map[string]string{
+				"testdata/all-panels.json": "dashboard1.json",
+			},
+			SkipResourceAssertions: true, // will check both at the same time below to reduce duration of this test
+		})
+	}()
+	go func() {
+		defer wg.Done()
+		helper.CreateRepo(t, TestRepo{
+			Name:   repo2,
+			Path:   path.Join(helper.ProvisioningPath, "repo2"),
+			Target: "folder",
+			Copies: map[string]string{
+				"testdata/timeline-demo.json": "dashboard2.json",
+			},
+			SkipResourceAssertions: true, // will check both at the same time below to reduce duration of this test
+		})
+	}()
+	wg.Wait()
 
-	// Create second repository targeting "folder" with its own subdirectory
-	helper.CreateRepo(t, TestRepo{
-		Name:   repo2,
-		Path:   path.Join(helper.ProvisioningPath, "repo2"),
-		Target: "folder",
-		Copies: map[string]string{
-			"testdata/timeline-demo.json": "dashboard2.json",
-		},
-		ExpectedDashboards: 2, // Total across both repos
-		ExpectedFolders:    2, // Total across both repos
-	})
+	require.EventuallyWithT(t, func(collect *assert.CollectT) {
+		dashboards, err := helper.DashboardsV1.Resource.List(t.Context(), metav1.ListOptions{})
+		if err != nil {
+			collect.Errorf("could not list dashboards error: %s", err.Error())
+			return
+		}
+		if len(dashboards.Items) != 2 {
+			collect.Errorf("should have the expected dashboards after sync. got: %d. expected: %d", len(dashboards.Items), 2)
+			return
+		}
+		folders, err := helper.Folders.Resource.List(t.Context(), metav1.ListOptions{})
+		if err != nil {
+			collect.Errorf("could not list folders: error: %s", err.Error())
+			return
+		}
+		if len(folders.Items) != 2 {
+			collect.Errorf("should have the expected folders after sync. got: %d. expected: %d", len(folders.Items), 2)
+			return
+		}
+
+		assert.Len(collect, dashboards.Items, 2)
+		assert.Len(collect, folders.Items, 2)
+	}, waitTimeoutDefault, waitIntervalDefault, "should have the expected dashboards and folders after sync")
 
 	// Test: Pull job should fail when trying to manage resources owned by another repository
 	t.Run("pull job should fail when trying to manage resources owned by another repository", func(t *testing.T) {
@@ -88,8 +116,8 @@ func TestIntegrationProvisioning_PullJobOwnershipProtection(t *testing.T) {
 		found := false
 		for _, errMsg := range jobObj.Status.Errors {
 			t.Logf("Error message: %s", errMsg)
-			if assert.Contains(t, errMsg, fmt.Sprintf("managed by repo '%s'", repo1)) &&
-				assert.Contains(t, errMsg, fmt.Sprintf("cannot be modified by repo '%s'", repo2)) {
+			if strings.Contains(errMsg, fmt.Sprintf("managed by repo '%s'", repo1)) &&
+				strings.Contains(errMsg, fmt.Sprintf("cannot be modified by repo '%s'", repo2)) {
 				found = true
 				break
 			}
@@ -130,14 +158,17 @@ func TestIntegrationProvisioning_PullJobOwnershipProtection(t *testing.T) {
 		persistentRepo2Dashboard, err := helper.DashboardsV1.Resource.Get(ctx, timelineUID, metav1.GetOptions{})
 		require.NoError(t, err, "repo2's dashboard should still exist after repo1 pull")
 		require.Equal(t, repo2, persistentRepo2Dashboard.GetAnnotations()[utils.AnnoKeyManagerIdentity], "ownership should remain with repo2")
-		require.Equal(t, repo2Dashboard.GetResourceVersion(), persistentRepo2Dashboard.GetResourceVersion(), "repo2's resource should not be modified by repo1 pull")
+		require.Equal(t, repo2Dashboard.GetGeneration(), persistentRepo2Dashboard.GetGeneration(), "repo2's resource should not be modified by repo1 pull")
 
 		// Step 4: Pull repo2 and verify repo1's resource is still intact
-		helper.SyncAndWait(t, repo2, nil)
+		helper.TriggerJobAndWaitForSuccess(t, repo2, provisioning.JobSpec{
+			Action: provisioning.JobActionPull,
+			Pull:   &provisioning.SyncJobOptions{},
+		})
 
 		persistentRepo1Dashboard, err := helper.DashboardsV1.Resource.Get(ctx, allPanelsUID, metav1.GetOptions{})
 		require.NoError(t, err, "repo1's dashboard should still exist after repo2 pull")
 		require.Equal(t, repo1, persistentRepo1Dashboard.GetAnnotations()[utils.AnnoKeyManagerIdentity], "ownership should remain with repo1")
-		require.Equal(t, repo1Dashboard.GetResourceVersion(), persistentRepo1Dashboard.GetResourceVersion(), "repo1's resource should not be modified by repo2 pull")
+		require.Equal(t, repo1Dashboard.GetGeneration(), persistentRepo1Dashboard.GetGeneration(), "repo1's resource should not be modified by repo2 pull")
 	})
 }