Remote Provisioning: Fix empty folder synchronization (#102789)

* Add behavior for hidden

* Rename to IsFilePathSupported

* Modify sync

* Revert renaming

* Fix the tests

* Add more tests

* Maintain empty folders until next full pulling

* Fix wording

* Consider the file as ignored

* Handle folder creation in sync

* Record folder creation / update

* Fix in manual test

* Ensure / slash for folders

* Refactor the tests

* Keep safe path

* Fix some cases

* Remove log lines

pkg/registry/apis/provisioning/jobs/sync/changes_test.go

@@ -1,7 +1,6 @@
 package sync
 
 import (
-	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -12,17 +11,25 @@ import (
 
 func TestChanges(t *testing.T) {
 	t.Run("start the same", func(t *testing.T) {
-		source, target := getBase(t)
+		source := []repository.FileTreeEntry{
+			{Path: "simplelocal/dashboard.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "simplelocal/dashboard.json", Hash: "xyz"},
+			},
+		}
+
 		changes, err := Changes(source, target)
 		require.NoError(t, err)
 		require.Empty(t, changes)
 	})
 
 	t.Run("create a source file", func(t *testing.T) {
-		source, target := getBase(t)
-		source = append(source, repository.FileTreeEntry{
-			Path: "muta.json", Hash: "xyz", Blob: true,
-		})
+		source := []repository.FileTreeEntry{
+			{Path: "muta.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{}
 
 		changes, err := Changes(source, target)
 		require.NoError(t, err)
@@ -33,9 +40,78 @@ func TestChanges(t *testing.T) {
 		}, changes[0])
 	})
 
+	t.Run("create empty folder structure for folders with unsupported file types", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "one/two/first.md", Hash: "xyz", Blob: true},
+			{Path: "other/second.md", Hash: "xyz", Blob: true},
+		}
+
+		target := &provisioning.ResourceList{}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Len(t, changes, 2)
+
+		require.Equal(t, ResourceFileChange{
+			Action: repository.FileActionCreated,
+			Path:   "one/two/",
+		}, changes[0])
+		require.Equal(t, ResourceFileChange{
+			Action: repository.FileActionCreated,
+			Path:   "other/",
+		}, changes[1])
+	})
+
+	t.Run("keep empty folders when unsupported file types are present", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "one/two/first.md", Hash: "xyz", Blob: true},
+			{Path: "other/second.md", Hash: "xyz", Blob: true},
+		}
+
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "one/two/", Resource: "folders"},
+				{Path: "other/", Resource: "folders"},
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes)
+	})
+	t.Run("keep common path to unsupported file types", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "common/first.md", Hash: "xyz", Blob: true},
+			{Path: "alsocommon/second.md", Hash: "xyz", Blob: true},
+		}
+
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "common/", Resource: "folders"},
+				{Path: "common/not-common/", Resource: "folders", Name: "uncommon-name", Hash: "xyz"},
+				{Path: "alsocommon/", Resource: "folders"},
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Len(t, changes, 1)
+		require.Equal(t, ResourceFileChange{
+			Action: repository.FileActionDeleted,
+			Path:   "common/not-common/",
+			Existing: &provisioning.ResourceListItem{
+				Path:     "common/not-common/",
+				Resource: "folders",
+				Name:     "uncommon-name",
+				Hash:     "xyz",
+			},
+		}, changes[0], "the uncommon path should be deleted")
+	})
+
 	t.Run("delete a source file", func(t *testing.T) {
-		source, target := getBase(t)
-		source = []repository.FileTreeEntry{source[0]}
+		source := []repository.FileTreeEntry{}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "adsl62h.yaml", Hash: "xyz", Group: "dashboard.grafana.app", Resource: "dashboards", Name: "adsl62h-hrw-f-fvlt2dghp-gufrc4lisksgmq-c"},
+			},
+		}
 
 		changes, err := Changes(source, target)
 		require.NoError(t, err)
@@ -48,7 +124,7 @@ func TestChanges(t *testing.T) {
 				Group:    "dashboard.grafana.app",
 				Resource: "dashboards",
 				Name:     "adsl62h-hrw-f-fvlt2dghp-gufrc4lisksgmq-c",
-				Hash:     "ce5d497c4deadde6831162ce8509e2b2b1776237",
+				Hash:     "xyz",
 			},
 		}, changes[0])
 	})
@@ -77,6 +153,7 @@ func TestChanges(t *testing.T) {
 		}
 		require.Equal(t, []string{
 			"zzz/longest/path/here.json", // not sorted yet
+			"x/y/z/",
 			"x/y/file.json",
 			"short/file.yml",
 			"a.json",
@@ -84,8 +161,20 @@ func TestChanges(t *testing.T) {
 	})
 
 	t.Run("modify a file", func(t *testing.T) {
-		source, target := getBase(t)
-		source[1].Hash = "different"
+		source := []repository.FileTreeEntry{
+			{Path: "adsl62h.yaml", Hash: "modified", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{
+					Path:     "adsl62h.yaml",
+					Group:    "dashboard.grafana.app",
+					Resource: "dashboards",
+					Name:     "adsl62h-hrw-f-fvlt2dghp-gufrc4lisksgmq-c",
+					Hash:     "original",
+				},
+			},
+		}
 
 		changes, err := Changes(source, target)
 		require.NoError(t, err)
@@ -98,48 +187,104 @@ func TestChanges(t *testing.T) {
 				Group:    "dashboard.grafana.app",
 				Resource: "dashboards",
 				Name:     "adsl62h-hrw-f-fvlt2dghp-gufrc4lisksgmq-c",
-				Hash:     "ce5d497c4deadde6831162ce8509e2b2b1776237",
+				Hash:     "original",
 			},
 		}, changes[0])
 	})
-}
 
-func getBase(t *testing.T) (source []repository.FileTreeEntry, target *provisioning.ResourceList) {
-	target = &provisioning.ResourceList{}
-	err := json.Unmarshal([]byte(`{
-		"kind": "ResourceList",
-		"apiVersion": "provisioning.grafana.app/v0alpha1",
-		"metadata": {},
-		"items": [
-			{
-				"path": "",
-				"group": "folder.grafana.app",
-				"resource": "folders",
-				"name": "simplelocal-3794ab9",
-				"hash": ""
+	t.Run("keep folder with hidden files", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "folder/.hidden.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "folder/", Resource: "folders"},
 			},
-			{
-				"path": "ad4lwp2.yaml",
-				"group": "dashboard.grafana.app",
-				"resource": "dashboards",
-				"name": "ad4lwp2-xofjsuo-mr5blr1zwimlfi0ds0pyrrpd",
-				"hash": "ca83d64b9c4a23fed975aacdf47e7de8878b4ae0"
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes, "folder should be kept when it contains hidden files")
+	})
+
+	t.Run("keep folder with invalid hidden paths", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "folder/.invalid/path.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "folder/", Resource: "folders"},
 			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes, "folder should be kept when it contains invalid hidden paths")
+	})
+
+	t.Run("keep folder with hidden folders", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "folder/.hidden/valid.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "folder/", Resource: "folders"},
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes, "folder should be kept when it contains hidden folders")
+	})
+
+	t.Run("unhidden path from hidden file", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "folder/.hidden/dashboard.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{
+			Items: []provisioning.ResourceListItem{
+				{Path: "folder/", Resource: "folders"},
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes, "hidden file should not be unhidden")
+	})
+	t.Run("hidden path to file", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "one/two/.hidden/dashboard.json", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{}
+		expected := []ResourceFileChange{
 			{
-				"path": "adsl62h.yaml",
-				"group": "dashboard.grafana.app",
-				"resource": "dashboards",
-				"name": "adsl62h-hrw-f-fvlt2dghp-gufrc4lisksgmq-c",
-				"hash": "ce5d497c4deadde6831162ce8509e2b2b1776237"
-			}
-		]
-	}`), target)
-	require.NoError(t, err)
+				Action: repository.FileActionCreated,
+				Path:   "one/two/",
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Equal(t, expected, changes)
+	})
+	t.Run("hidden path to folder", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: "one/two/.hidden/folder/", Hash: "xyz", Blob: true},
+		}
+		target := &provisioning.ResourceList{}
+		expected := []ResourceFileChange{
+			{
+				Action: repository.FileActionCreated,
+				Path:   "one/two/",
+			},
+		}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Equal(t, expected, changes)
+	})
+	t.Run("hidden at the root", func(t *testing.T) {
+		source := []repository.FileTreeEntry{
+			{Path: ".hidden/dashboard.json", Hash: "xyz", Blob: true},
+		}
 
-	source = []repository.FileTreeEntry{
-		{Path: "ad4lwp2.yaml", Hash: "ca83d64b9c4a23fed975aacdf47e7de8878b4ae0", Blob: true},
-		{Path: "adsl62h.yaml", Hash: "ce5d497c4deadde6831162ce8509e2b2b1776237", Blob: true},
-	}
-
-	return // named values!
+		target := &provisioning.ResourceList{}
+		changes, err := Changes(source, target)
+		require.NoError(t, err)
+		require.Empty(t, changes)
+	})
 }