[PROVISIONING] Support empty folders (#99560)

* Support empty folders * Make it work * Add FIXMEs * Fix some issues * Move logic to tree * Further refactoring * Further refactoring * fix issue with root folder * Further refactoring * Fix bug with nested * Add Walk Function * Move walk to separate file * Add comment * Move walk to safepath * Fix bug for parent folder * Use the tree * Rename argument in walk function * WIP: walk over the tree * Filter out unprovisioned folders * Use the annotation instead * Fix lint issue * Implement basic tree walk * Use v1alpha1 for dashboards * Fix local provisioning * Fix folder creation local * Fix styling issue * Revert "Fix styling issue" This reverts commit b0f77a8f74. * Update pkg/registry/apis/provisioning/jobs/exporter.go Co-authored-by: Mariell Hoversholm <mariell.hoversholm@grafana.com> --------- Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Mariell Hoversholm <mariell.hoversholm@grafana.com>
2025-01-29 20:42:08 +03:00
1 parent 0fece88980
commit ef300cc878
@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"path"
 	"strings"
+	"time"

 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -73,7 +74,9 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 		}

 		filePath := strings.TrimPrefix(r.URL.Path[idx+len(prefix):], "/")
-		if filePath == "" || strings.HasSuffix(filePath, "/") {
+		isFolderPath := strings.HasSuffix(filePath, "/")
+
+		if r.Method == http.MethodGet && (filePath == "" || isFolderPath) {
 			if len(filePath) > 0 {
 				responder.Error(apierrors.NewBadRequest("folder navigation not yet supported"))
 				return
@@ -100,13 +103,15 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 			return
 		}

-		switch path.Ext(filePath) {
-		case ".json", ".yaml", ".yml":
-			// ok
-		default:
-			logger.Debug("got a file extension that was not JSON or YAML", "extension", path.Ext(filePath))
-			responder.Error(apierrors.NewBadRequest("only yaml and json files supported"))
-			return
+		if !isFolderPath {
+			switch path.Ext(filePath) {
+			case ".json", ".yaml", ".yml":
+				// ok
+			default:
+				logger.Debug("got a file extension that was not JSON or YAML", "extension", path.Ext(filePath))
+				responder.Error(apierrors.NewBadRequest("only yaml and json files supported"))
+				return
+			}
 		}

 		var obj *provisioning.ResourceWrapper
@@ -117,7 +122,11 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 		case http.MethodPost:
 			obj, err = s.doWrite(ctx, false, repo, filePath, ref, message, r)
 		case http.MethodPut:
-			obj, err = s.doWrite(ctx, true, repo, filePath, ref, message, r)
+			if isFolderPath {
+				err = apierrors.NewMethodNotSupported(provisioning.RepositoryResourceInfo.GroupResource(), r.Method)
+			} else {
+				obj, err = s.doWrite(ctx, true, repo, filePath, ref, message, r)
+			}
 		case http.MethodDelete:
 			obj, err = s.doDelete(ctx, repo, filePath, ref, message)
 		default:
@@ -182,6 +191,20 @@ func (s *filesConnector) doWrite(ctx context.Context, update bool, repo reposito
 	}

 	defer func() { _ = req.Body.Close() }()
+	if strings.HasSuffix(path, "/") {
+		if err := repo.Create(ctx, path, ref, nil, message); err != nil {
+			return nil, fmt.Errorf("failed to create folder: %w", err)
+		}
+		return &provisioning.ResourceWrapper{
+			Path:      path,
+			Ref:       ref,
+			Timestamp: &metav1.Time{Time: time.Now()},
+			// TODO: should we return something here?
+			// TypeMeta:  metav1.TypeMeta{},
+			// Resource: provisioning.ResourceObjects{},
+		}, nil
+	}
+
 	data, err := io.ReadAll(req.Body)
 	if err != nil {
 		return nil, err
@@ -9,11 +9,13 @@ import (
 	"gopkg.in/yaml.v3"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"

 	"github.com/grafana/grafana-app-sdk/logging"
 	apiutils "github.com/grafana/grafana/pkg/apimachinery/utils"
+	folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
@@ -29,6 +31,7 @@ type Exporter interface {

 type exporter struct {
 	client     *resources.DynamicClient
+	dashboards dynamic.ResourceInterface
 	folders    dynamic.ResourceInterface
 	repository repository.Repository
 }
@@ -39,13 +42,21 @@ func NewExporter(
 ) (Exporter, error) {
 	dynamicClient := parser.Client()
 	folders := dynamicClient.Resource(schema.GroupVersionResource{
-		Group:    "folder.grafana.app",
-		Version:  "v0alpha1",
-		Resource: "folders",
+		Group:    folders.GROUP,
+		Version:  folders.VERSION,
+		Resource: folders.RESOURCE,
 	})
+
+	dashboards := dynamicClient.Resource(schema.GroupVersionResource{
+		Group:    "dashboard.grafana.app",
+		Version:  "v1alpha1",
+		Resource: "dashboards",
+	})
+
 	return &exporter{
 		client:     dynamicClient,
 		folders:    folders,
+		dashboards: dashboards,
 		repository: repo,
 	}, nil
 }
@@ -56,12 +67,6 @@ func (r *exporter) Export(ctx context.Context,
 	progress func(provisioning.JobStatus) error,
 ) (*provisioning.JobStatus, error) {
 	logger := logging.FromContext(ctx)
-	dashboardIface := r.client.Resource(schema.GroupVersionResource{
-		Group:    "dashboard.grafana.app",
-		Version:  "v2alpha1",
-		Resource: "dashboards",
-	})
-
 	err := progress(provisioning.JobStatus{
 		State:   provisioning.JobStateWorking,
 		Message: "getting folder tree...",
@@ -69,12 +74,64 @@ func (r *exporter) Export(ctx context.Context,
 	if err != nil {
 		return nil, err
 	}
+	var ref string
+	if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType {
+		ref = r.repository.Config().Spec.GitHub.Branch
+	}
+	ns := r.repository.Config().GetNamespace()
+	logger = logger.With("ref", ref, "namespace", ns)

-	folders, err := resources.FetchRepoFolderTree(ctx, r.client)
+	err = progress(provisioning.JobStatus{
+		State:   provisioning.JobStateWorking,
+		Message: "exporting folders...",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: handle pagination
+	rawFolders, err := r.folders.List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to list folders: %w", err)
 	}

+	var unprovisionedFolders []unstructured.Unstructured
+	for _, f := range rawFolders.Items {
+		repoName := f.GetAnnotations()[apiutils.AnnoKeyRepoName]
+		if repoName == repo.Config().GetName() {
+			logger.Info("skip as folder is already in repository", "folder", f.GetName())
+			continue
+		}
+
+		unprovisionedFolders = append(unprovisionedFolders, f)
+	}
+
+	folderTree := resources.NewFolderTreeFromUnstructure(ctx, unprovisionedFolders)
+	err = folderTree.Walk(ctx, func(ctx context.Context, folder resources.Folder) error {
+		p := folder.Path + "/"
+		logger := logger.With("path", p)
+
+		_, err = r.repository.Read(ctx, p, ref)
+		if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
+			logger.Error("failed to check if folder exists before writing", "error", err)
+			return fmt.Errorf("failed to check if folder exists before writing: %w", err)
+		} else if err == nil {
+			logger.Info("folder already exists")
+			return nil
+		}
+
+		// ErrFileNotFound
+		if err := r.repository.Create(ctx, p, ref, nil, "export of folder `"+p+"` in namespace "+ns); err != nil {
+			logger.Error("failed to write a folder in repository", "error", err)
+			return fmt.Errorf("failed to write folder in repo: %w", err)
+		}
+		logger.Debug("successfully exported folder")
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to write folders: %w", err)
+	}
+
 	err = progress(provisioning.JobStatus{
 		State:   provisioning.JobStateWorking,
 		Message: "writing dashboards...",
@@ -83,7 +140,7 @@ func (r *exporter) Export(ctx context.Context,
 		return nil, err
 	}
 	// TODO: handle pagination
-	dashboardList, err := dashboardIface.List(ctx, metav1.ListOptions{Limit: 1000})
+	dashboardList, err := r.dashboards.List(ctx, metav1.ListOptions{Limit: 1000})
 	if err != nil {
 		return nil, fmt.Errorf("failed to list dashboards: %w", err)
 	}
@@ -98,16 +155,25 @@ func (r *exporter) Export(ctx context.Context,
 		logger := logger.With("item", name)
 		ns := r.repository.Config().GetNamespace()
 		if item.GetNamespace() != ns {
-			logger.Debug("skipping dashboard item due to mismatching namespace", "got", ns)
+			// This case shouldn't happen
+			logger.Error("skipping dashboard item due to mismatching namespace", "got", ns)
+			continue
+		}
+
+		repoName := item.GetAnnotations()[apiutils.AnnoKeyRepoName]
+		if repoName == repo.Config().GetName() {
+			logger.Info("skip as dashboard is already in repository", "name", name)
 			continue
 		}

 		folder := item.GetAnnotations()[apiutils.AnnoKeyFolder]
 		logger = logger.With("folder", folder)
-		fid, ok := folders.DirPath(folder, r.repository.Config().Spec.Folder)
+
+		// Get the absolute path of the folder
+		fid, ok := folderTree.DirPath(folder, "")
 		if !ok {
-			logger.Debug("folder of item was not in tree of repository")
-			continue
+			logger.Error("folder of item was not in tree of repository")
+			return nil, fmt.Errorf("folder of item was not in tree of repository")
 		}

 		delete(item.Object, "metadata")
@@ -118,12 +184,6 @@ func (r *exporter) Export(ctx context.Context,
 		fileName := path.Join(fid.Path, name+".yaml")
 		logger = logger.With("file", fileName)

-		var ref string
-		if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType {
-			ref = r.repository.Config().Spec.GitHub.Branch
-		}
-		logger = logger.With("ref", ref)
-
 		_, err = r.repository.Read(ctx, fileName, ref)
 		if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
 			logger.Error("failed to check if file exists before writing", "error", err)
@@ -23,6 +23,7 @@ import (
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
+	"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
 )

 type Syncer struct {
@@ -61,8 +62,23 @@ func (r *Syncer) Sync(ctx context.Context, complete bool) (string, error) {
 	lastCommit := cfg.Status.Sync.Hash
 	versionedRepo, isVersioned := r.repository.(repository.VersionedRepository)

-	if err := r.ensureRepositoryFolderExists(ctx); err != nil {
-		return "", fmt.Errorf("ensure repository folder exists: %w", err)
+	// Ensure the configured folder exists and it's managed by the repository
+	if cfg.Spec.Folder != "" {
+		title := cfg.Spec.Title
+		if title == "" {
+			title = cfg.Spec.Folder
+		}
+
+		folder := resources.Folder{
+			Path:  "",
+			ID:    cfg.Spec.Folder,
+			Title: title,
+		}
+
+		// If the folder already exists, the parent won't be changed
+		if err := r.ensureFolderExists(ctx, folder, ""); err != nil {
+			return "", fmt.Errorf("ensure repository folder exists: %w", err)
+		}
 	}

 	logger := logging.FromContext(ctx)
@@ -181,28 +197,46 @@ func (r *Syncer) deleteListResource(ctx context.Context, item provisioning.Resou

 // replicateTree replicates all files in the repository.
 func (r *Syncer) replicateTree(ctx context.Context, ref string) error {
-	tree, err := r.repository.ReadTree(ctx, ref)
+	// Load the tree of the repository
+	fileTree, err := r.repository.ReadTree(ctx, ref)
 	if err != nil {
 		return fmt.Errorf("read tree: %w", err)
 	}

+	// Load the list of resources in the cluster
 	list, err := r.lister.List(ctx, r.client.GetNamespace(), r.repository.Config().Name)
 	if err != nil {
 		return fmt.Errorf("list resources: %w", err)
 	}

-	if err := r.cleanUnnecessaryResources(ctx, tree, list); err != nil {
+	// Remove resources not longer present in the repository
+	if err := r.cleanUnnecessaryResources(ctx, fileTree, list); err != nil {
 		return fmt.Errorf("clean up before replicating tree: %w", err)
 	}

+	// Create folders
+	folderTree := resources.NewFolderTreeFromResourceList(list)
+	for _, entry := range fileTree {
+		parent := resources.ParentFolder(path.Dir(entry.Path), r.repository.Config())
+		if err := r.ensureFolderPathExists(ctx, path.Dir(entry.Path), parent, folderTree); err != nil {
+			return fmt.Errorf("ensure folder path exists: %w", err)
+		}
+	}
+
+	// Create a list of hashes to identify changed files
 	hashes := make(map[string]string)
 	for _, item := range list.Items {
 		hashes[item.Path] = item.Hash
 	}

-	// FIXME: Add support for empty folders
-	for _, entry := range tree {
+	// Create Dashboards
+	for _, entry := range fileTree {
 		logger := logging.FromContext(ctx).With("file", entry.Path)
+		if hashes[entry.Path] == entry.Hash {
+			logger.Debug("file is up to date")
+			continue
+		}
+
 		if !entry.Blob {
 			logger.Debug("ignoring non-blob entry")
 			continue
@@ -213,11 +247,6 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error {
 			continue
 		}

-		if hashes[entry.Path] == entry.Hash {
-			logger.Debug("file is up to date")
-			continue
-		}
-
 		info, err := r.repository.Read(ctx, entry.Path, ref)
 		if err != nil {
 			return fmt.Errorf("read file: %w", err)
@@ -227,7 +256,12 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error {
 		info.Hash = entry.Hash
 		info.Modified = nil // modified?

-		if err := r.replicateFile(ctx, info); err != nil {
+		parsed, err := r.parseResource(ctx, info)
+		if err != nil {
+			return fmt.Errorf("parse resource: %w", err)
+		}
+
+		if err := r.replicateFile(ctx, parsed, folderTree); err != nil {
 			if errors.Is(err, resources.ErrUnableToReadResourceBytes) {
 				logger.Info("file does not contain a resource")
 				continue
@@ -241,21 +275,16 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error {

 // replicateFile creates a new resource in the cluster.
 // If the resource already exists, it will be updated.
-func (r *Syncer) replicateFile(ctx context.Context, fileInfo *repository.FileInfo) error {
-	logger := logging.FromContext(ctx).With("file", fileInfo.Path, "ref", fileInfo.Ref)
-	file, err := r.parseResource(ctx, fileInfo)
-	if err != nil {
-		return err
-	}
+func (r *Syncer) replicateFile(ctx context.Context, file *resources.ParsedResource, folderTree *resources.FolderTree) error {
+	logger := logging.FromContext(ctx).With("file", file.Info.Path, "ref", file.Info.Ref)
 	logger = logger.With("action", file.Action, "name", file.Obj.GetName(), "file_namespace", file.Obj.GetNamespace(), "namespace", r.client.GetNamespace())

-	parent, err := r.createFolderPath(ctx, fileInfo.Path)
-	if err != nil {
-		return fmt.Errorf("failed to create folder path: %w", err)
-	}
+	parent := resources.ParentFolder(file.Info.Path, r.repository.Config())
 	logger = logger.With("folder", parent)
-
 	if parent != "" {
+		if !folderTree.In(parent) {
+			return fmt.Errorf("failed to find parent in tree for %s in %s", file.Info.Path, parent)
+		}
 		file.Meta.SetFolder(parent)
 	}

@@ -303,71 +332,24 @@ func (r *Syncer) replicateFile(ctx context.Context, fileInfo *repository.FileInf
 	return nil
 }

-func (r *Syncer) createFolderPath(ctx context.Context, filePath string) (string, error) {
-	dir := path.Dir(filePath)
-	parent := r.repository.Config().Spec.Folder
-	if dir == "." || dir == "/" {
-		return parent, nil
-	}
-
-	logger := logging.FromContext(ctx).With("file", filePath)
-
-	var currentPath string
-	for _, folder := range strings.Split(dir, "/") {
-		if folder == "" {
-			// Trailing / leading slash?
-			continue
-		}
-
-		currentPath = path.Join(currentPath, folder)
-		folderID := resources.ParseFolder(currentPath, r.repository.Config().GetName())
-
-		logger := logger.With("folder", currentPath)
-		obj, err := r.folders.Get(ctx, folderID.ID, metav1.GetOptions{})
-		// FIXME: Check for IsNotFound properly
-		if obj != nil || err == nil {
-			logger.Debug("folder already existed")
-			parent = folderID.ID
-			continue
-		}
-
-		obj = &unstructured.Unstructured{
-			Object: map[string]interface{}{
-				"spec": map[string]any{
-					"title":       folderID.Title,
-					"description": "Repository-managed folder.",
-				},
-			},
-		}
-
-		meta, err := utils.MetaAccessor(obj)
-		if err != nil {
-			return "", fmt.Errorf("create meta accessor for the object: %w", err)
-		}
-
-		obj.SetNamespace(r.client.GetNamespace())
-		obj.SetName(folderID.ID)
-		meta.SetFolder(parent)
-		meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
-			Name:      r.repository.Config().Name,
-			Path:      currentPath,
-			Hash:      "",  // FIXME: which hash?
-			Timestamp: nil, // ???&info.Modified.Time,
-		})
-
-		_, err = r.folders.Create(ctx, obj, metav1.CreateOptions{})
-		if err != nil {
-			return parent, fmt.Errorf("failed to create folder '%s': %w", folderID.ID, err)
-		}
-
-		parent = folderID.ID
-		logger.Info("folder created")
-	}
-
-	return parent, nil
-}
-
 func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.FileChange) error {
+	// Create an empty tree to avoid loading all folders unnecessarily
+	folderTree := resources.NewEmptyFolderTree()
+
+	// Create folder structure first
+	for _, change := range changes {
+		if change.Action == repository.FileActionDeleted {
+			// FIXME: this will leave empty folder behind until the next sync
+			continue
+		}
+
+		parent := resources.ParentFolder(path.Dir(change.Path), r.repository.Config())
+		if err := r.ensureFolderPathExists(ctx, path.Dir(change.Path), parent, folderTree); err != nil {
+			return fmt.Errorf("ensure folder path exists: %w", err)
+		}
+	}
+
+	// Replicate the file changes
 	for _, change := range changes {
 		if resources.ShouldIgnorePath(change.Path) {
 			continue
@@ -378,9 +360,14 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File
 			return fmt.Errorf("read file: %w", err)
 		}

+		parsed, err := r.parseResource(ctx, fileInfo)
+		if err != nil {
+			return fmt.Errorf("parse resource: %w", err)
+		}
+
 		switch change.Action {
 		case repository.FileActionCreated, repository.FileActionUpdated:
-			if err := r.replicateFile(ctx, fileInfo); err != nil {
+			if err := r.replicateFile(ctx, parsed, folderTree); err != nil {
 				return fmt.Errorf("replicate file: %w", err)
 			}
 		case repository.FileActionRenamed:
@@ -389,15 +376,19 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File
 			if err != nil {
 				return fmt.Errorf("read previous path: %w", err)
 			}
-			if err := r.deleteFile(ctx, oldPath); err != nil {
+			oldParsed, err := r.parseResource(ctx, oldPath)
+			if err != nil {
+				return fmt.Errorf("parse old resource: %w", err)
+			}
+			if err := r.deleteFile(ctx, oldParsed); err != nil {
 				return fmt.Errorf("delete file: %w", err)
 			}

-			if err := r.replicateFile(ctx, fileInfo); err != nil {
+			if err := r.replicateFile(ctx, parsed, folderTree); err != nil {
 				return fmt.Errorf("replicate file in new path: %w", err)
 			}
 		case repository.FileActionDeleted:
-			if err := r.deleteFile(ctx, fileInfo); err != nil {
+			if err := r.deleteFile(ctx, parsed); err != nil {
 				return fmt.Errorf("delete file: %w", err)
 			}
 		}
@@ -406,13 +397,8 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File
 	return nil
 }

-func (r *Syncer) deleteFile(ctx context.Context, fileInfo *repository.FileInfo) error {
-	file, err := r.parseResource(ctx, fileInfo)
-	if err != nil {
-		return err
-	}
-
-	_, err = file.Client.Get(ctx, file.Obj.GetName(), metav1.GetOptions{})
+func (r *Syncer) deleteFile(ctx context.Context, file *resources.ParsedResource) error {
+	_, err := file.Client.Get(ctx, file.Obj.GetName(), metav1.GetOptions{})
 	// FIXME: Remove the 'false &&' when .Get returns 404 on 404 instead of 500. Until then, this is a really ugly workaround.
 	if false && err != nil && !apierrors.IsNotFound(err) {
 		return fmt.Errorf("failed to check if object already exists: %w", err)
@@ -446,25 +432,59 @@ func (r *Syncer) parseResource(ctx context.Context, fileInfo *repository.FileInf
 	return file, nil
 }

-func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error {
-	if r.repository.Config().Spec.Folder == "" {
-		return nil
-	}
+// ensureFolderPathExists creates the folder structure in the cluster.
+func (r *Syncer) ensureFolderPathExists(ctx context.Context, dirPath, parent string, folderTree *resources.FolderTree) error {
+	return safepath.Walk(ctx, dirPath, func(ctx context.Context, path string) error {
+		fid := resources.ParseFolder(path, r.repository.Config().GetName())
+		if folderTree.In(fid.ID) {
+			// already visited
+			parent = fid.ID
+			return nil
+		}

-	obj, err := r.folders.Get(ctx, r.repository.Config().Spec.Folder, metav1.GetOptions{})
+		if err := r.ensureFolderExists(ctx, fid, parent); err != nil {
+			return fmt.Errorf("ensure folder exists: %w", err)
+		}
+
+		folderTree.Add(fid, parent)
+		parent = fid.ID
+
+		return nil
+	})
+}
+
+// ensureFolderExists creates the folder if it doesn't exist.
+// If the folder already exists:
+// - it will update the repository info.
+// - it will keep the parent folder already set
+func (r *Syncer) ensureFolderExists(ctx context.Context, folder resources.Folder, parent string) error {
+	cfg := r.repository.Config()
+	repoInfo := &utils.ResourceRepositoryInfo{
+		Name:      cfg.GetName(),
+		Path:      folder.Path,
+		Hash:      "",  // FIXME: which hash?
+		Timestamp: nil, // ???&info.Modified.Time,
+	}
+	obj, err := r.folders.Get(ctx, folder.ID, metav1.GetOptions{})
 	if err == nil {
 		meta, err := utils.MetaAccessor(obj)
 		if err != nil {
 			return fmt.Errorf("create meta accessor for the object: %w", err)
 		}

-		meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
-			Name:      r.repository.Config().Name,
-			Path:      "",
-			Hash:      "",  // FIXME: which hash?
-			Timestamp: nil, // ???&info.Modified.Time,
-		})
+		existingInfo, err := meta.GetRepositoryInfo()
+		if err != nil {
+			return fmt.Errorf("get repository info: %w", err)
+		}

+		// Skip update if already present with right values
+		if existingInfo.Name == repoInfo.Name && existingInfo.Path == repoInfo.Path {
+			return nil
+		}
+
+		meta.SetRepositoryInfo(repoInfo)
+
+		// TODO: should we keep the parent?
 		if _, err := r.folders.Update(ctx, obj, metav1.UpdateOptions{}); err != nil {
 			return fmt.Errorf("failed to add repo info to configured folder: %w", err)
 		}
@@ -474,16 +494,10 @@ func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error {
 		return fmt.Errorf("failed to check if folder exists: %w", err)
 	}

-	cfg := r.repository.Config()
-	title := cfg.Spec.Title
-	if title == "" {
-		title = cfg.Spec.Folder
-	}
-
 	obj = &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"spec": map[string]any{
-				"title": title, // TODO: how do we want to get this?
+				"title": folder.Title,
 			},
 		},
 	}
@@ -494,13 +508,11 @@ func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error {
 	}

 	obj.SetNamespace(cfg.GetNamespace())
-	obj.SetName(cfg.Spec.Folder)
-	meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
-		Name:      r.repository.Config().Name,
-		Path:      "",
-		Hash:      "",  // FIXME: which hash?
-		Timestamp: nil, // ???&info.Modified.Time,
-	})
+	obj.SetName(folder.ID)
+	if parent != "" {
+		meta.SetFolder(parent)
+	}
+	meta.SetRepositoryInfo(repoInfo)

 	if _, err := r.folders.Create(ctx, obj, metav1.CreateOptions{}); err != nil {
 		return fmt.Errorf("failed to create folder: %w", err)
@@ -160,7 +160,10 @@ func (r *githubRepository) Read(ctx context.Context, filePath, ref string) (*Fil
 		return nil, fmt.Errorf("get contents: %w", err)
 	}
 	if dirContent != nil {
-		return nil, fmt.Errorf("input path was a directory")
+		return &FileInfo{
+			Path: filePath,
+			Ref:  ref,
+		}, nil
 	}

 	data, err := content.GetFileContent()
@@ -224,6 +227,16 @@ func (r *githubRepository) Create(ctx context.Context, path, ref string, data []
 	owner := r.config.Spec.GitHub.Owner
 	repo := r.config.Spec.GitHub.Repository

+	// Create .keep file if it is a directory
+	if strings.HasSuffix(path, "/") {
+		if data != nil {
+			return apierrors.NewBadRequest("data cannot be provided for a directory")
+		}
+
+		path = strings.TrimSuffix(path, "/") + "/.keep"
+		data = []byte{}
+	}
+
 	err := r.gh.CreateFile(ctx, owner, repo, path, ref, comment, data)
 	if errors.Is(err, pgh.ErrResourceAlreadyExists) {
 		return &apierrors.StatusError{
@@ -262,6 +275,9 @@ func (r *githubRepository) Update(ctx context.Context, path, ref string, data []

 		return fmt.Errorf("get content before file update: %w", err)
 	}
+	if file.IsDirectory() {
+		return apierrors.NewBadRequest("cannot update a directory")
+	}

 	if err := r.gh.UpdateFile(ctx, owner, repo, path, ref, comment, file.GetSHA(), data); err != nil {
 		return fmt.Errorf("update file: %w", err)
@@ -279,10 +295,14 @@ func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string
 		return fmt.Errorf("create branch on delete: %w", err)
 	}

+	return r.deleteRecursively(ctx, path, ref, comment)
+}
+
+func (r *githubRepository) deleteRecursively(ctx context.Context, path, ref, comment string) error {
 	owner := r.config.Spec.GitHub.Owner
 	repo := r.config.Spec.GitHub.Repository

-	file, _, err := r.gh.GetContents(ctx, owner, repo, path, ref)
+	file, contents, err := r.gh.GetContents(ctx, owner, repo, path, ref)
 	if err != nil {
 		if errors.Is(err, pgh.ErrResourceNotFound) {
 			return &apierrors.StatusError{
@@ -295,7 +315,24 @@ func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string
 		return fmt.Errorf("finding file to delete: %w", err)
 	}

-	return r.gh.DeleteFile(ctx, owner, repo, path, ref, comment, file.GetSHA())
+	if file != nil && !file.IsDirectory() {
+		return r.gh.DeleteFile(ctx, owner, repo, path, ref, comment, file.GetSHA())
+	}
+
+	for _, c := range contents {
+		if c.IsDirectory() {
+			if err := r.deleteRecursively(ctx, c.GetPath(), ref, comment); err != nil {
+				return fmt.Errorf("delete file recursive: %w", err)
+			}
+			continue
+		}
+
+		if err := r.gh.DeleteFile(ctx, owner, repo, c.GetPath(), ref, comment, c.GetSHA()); err != nil {
+			return fmt.Errorf("delete file: %w", err)
+		}
+	}
+
+	return nil
 }

 func (r *githubRepository) History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error) {
@@ -473,6 +510,9 @@ func (r *githubRepository) parsePushEvent(event *github.PushEvent) (*provisionin
 		Job: &provisioning.JobSpec{
 			Repository: r.Config().GetName(),
 			Action:     provisioning.JobActionSync,
+			Sync: &provisioning.SyncOptions{
+				Complete: false,
+			},
 		},
 	}, nil
 }
@@ -191,29 +191,39 @@ func (r *localRepository) Read(ctx context.Context, filePath string, ref string)

 	filePath, err := safepath.Join(r.path, filePath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("join path: %w", err)
 	}

 	info, err := os.Stat(filePath)
 	if errors.Is(err, os.ErrNotExist) {
 		return nil, ErrFileNotFound
 	} else if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("stat file: %w", err)
+	}
+
+	path := strings.TrimPrefix(filePath, safepath.Clean(r.path))
+	if info.IsDir() {
+		return &FileInfo{
+			Path: path,
+			Modified: &metav1.Time{
+				Time: info.ModTime(),
+			},
+		}, nil
 	}

 	//nolint:gosec
 	data, err := os.ReadFile(filePath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("read file: %w", err)
 	}

 	hash, _, err := r.calculateFileHash(filePath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("calculate hash of file: %w", err)
 	}

 	return &FileInfo{
-		Path: strings.TrimPrefix(filePath, safepath.Clean(r.path)),
+		Path: path,
 		Data: data,
 		Hash: hash,
 		Modified: &metav1.Time{
@@ -281,11 +291,13 @@ func (r *localRepository) Create(ctx context.Context, sanitisedPath string, ref
 		return err
 	}

+	// before sanitizing, check if the path is a directory
+	isDir := strings.HasSuffix(sanitisedPath, "/")
 	inputUnsafePath := sanitisedPath // do NOT use in path operations, only for responses!

 	sanitisedPath, err := safepath.Join(r.path, sanitisedPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("join path: %w", err)
 	}

 	_, err = os.Stat(sanitisedPath)
@@ -296,8 +308,20 @@ func (r *localRepository) Create(ctx context.Context, sanitisedPath string, ref
 		return apierrors.NewAlreadyExists(provisioning.RepositoryResourceInfo.GroupResource(), inputUnsafePath)
 	}

+	if isDir {
+		if data != nil {
+			return apierrors.NewBadRequest("data cannot be provided for a directory")
+		}
+
+		if err := os.MkdirAll(sanitisedPath, 0700); err != nil {
+			return apierrors.NewInternalError(fmt.Errorf("failed to create path: %w", err))
+		}
+
+		return nil
+	}
+
 	if err := os.MkdirAll(path.Dir(sanitisedPath), 0700); err != nil {
-		return apierrors.NewInternalError(fmt.Errorf("failed to create file: %w", err))
+		return apierrors.NewInternalError(fmt.Errorf("failed to create path: %w", err))
 	}

 	return os.WriteFile(sanitisedPath, data, 0600)
@@ -310,7 +334,11 @@ func (r *localRepository) Update(ctx context.Context, path string, ref string, d

 	path, err := safepath.Join(r.path, path)
 	if err != nil {
-		return err
+		return fmt.Errorf("join path: %w", err)
+	}
+
+	if strings.HasSuffix(path, "/") {
+		return apierrors.NewBadRequest("cannot update a directory")
 	}

 	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
@@ -5,6 +5,8 @@ import (
 	"encoding/base64"
 	"path"
 	"strings"
+
+	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 )

 // sanitiseKubeName removes all characters that don't fulfil the DNS subdomain name rules: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
@@ -131,6 +133,14 @@ func ParseFolder(dirPath, repositoryName string) Folder {
 	}
 }

+func ParentFolder(filePath string, repository *provisioning.Repository) string {
+	parent := path.Dir(filePath)
+	if parent == "." || parent == "/" {
+		return repository.Spec.Folder
+	}
+	return ParseFolder(parent, repository.GetName()).ID
+}
+
 func folderTitleFromPath(cleanDirPath string) string {
 	titleIdx := strings.LastIndex(cleanDirPath, "/")
 	title := cleanDirPath
@@ -3,11 +3,13 @@ package resources
 import (
 	"context"
 	"path"
+	"sort"
+	"strings"

 	apiutils "github.com/grafana/grafana/pkg/apimachinery/utils"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
+	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 )

 // FolderTree contains the entire set of folders (at a given snapshot in time) of the Grafana instance.
@@ -60,22 +62,46 @@ func (t *FolderTree) DirPath(folder, baseFolder string) (fid Folder, ok bool) {
 	return fid, ok
 }

-func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTree, error) {
-	iface := client.Resource(schema.GroupVersionResource{
-		Group:    "folder.grafana.app",
-		Version:  "v0alpha1",
-		Resource: "folders",
-	})
+func (t *FolderTree) Add(folder Folder, parent string) {
+	t.tree[folder.ID] = parent
+	t.folders[folder.ID] = folder
+}

-	// TODO: handle pagination
-	rawFolders, err := iface.List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return nil, err
+type WalkFunc func(ctx context.Context, folder Folder) error
+
+func (t *FolderTree) Walk(ctx context.Context, fn WalkFunc) error {
+	toWalk := make([]Folder, 0, len(t.folders))
+	for _, folder := range t.folders {
+		folder, _ := t.DirPath(folder.ID, "")
+		toWalk = append(toWalk, folder)
 	}

-	tree := make(map[string]string, len(rawFolders.Items))
-	folders := make(map[string]Folder, len(rawFolders.Items))
-	for _, rf := range rawFolders.Items {
+	// sort by depth of the paths
+	sort.Slice(toWalk, func(i, j int) bool {
+		return len(strings.Split(toWalk[i].Path, "/")) < len(strings.Split(toWalk[j].Path, "/"))
+	})
+
+	for _, folder := range toWalk {
+		if err := fn(ctx, folder); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewEmptyFolderTree() *FolderTree {
+	return &FolderTree{
+		tree:    make(map[string]string, 0),
+		folders: make(map[string]Folder, 0),
+	}
+}
+
+func NewFolderTreeFromUnstructure(ctx context.Context, rawFolders []unstructured.Unstructured) *FolderTree {
+	tree := make(map[string]string, len(rawFolders))
+	folders := make(map[string]Folder, len(rawFolders))
+
+	for _, rf := range rawFolders {
 		name := rf.GetName()
 		// TODO: Can I use MetaAccessor here?
 		parent := rf.GetAnnotations()[apiutils.AnnoKeyFolder]
@@ -84,7 +110,8 @@ func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTre
 		id := Folder{
 			Title: name,
 			ID:    name,
-			Path:  "", // We'll set this later in the DirPath function :)
+			// TODO: should not this be be the annotation itself?
+			Path: "", // We'll set this later in the DirPath function :)
 		}
 		if title, ok, _ := unstructured.NestedString(rf.Object, "spec", "title"); ok {
 			// If the title doesn't exist (it should), we'll just use the K8s name.
@@ -96,5 +123,27 @@ func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTre
 	return &FolderTree{
 		tree:    tree,
 		folders: folders,
-	}, nil
+	}
+}
+
+func NewFolderTreeFromResourceList(resources *provisioning.ResourceList) *FolderTree {
+	tree := make(map[string]string, len(resources.Items))
+	folderIDs := make(map[string]Folder, len(resources.Items))
+	for _, rf := range resources.Items {
+		if rf.Group != folders.GROUP {
+			continue
+		}
+
+		tree[rf.Name] = rf.Folder
+		folderIDs[rf.Name] = Folder{
+			Title: rf.Title,
+			ID:    rf.Name,
+			Path:  rf.Path,
+		}
+	}
+
+	return &FolderTree{
+		tree,
+		folderIDs,
+	}
 }
@@ -0,0 +1,31 @@
+package safepath
+
+import (
+	"context"
+	"path"
+	"strings"
+)
+
+type WalkFunc func(ctx context.Context, path string) error
+
+// Walk walks the given folder path and calls the given function for each folder.
+func Walk(ctx context.Context, p string, fn WalkFunc) error {
+	if p == "." || p == "/" {
+		return nil
+	}
+
+	var currentPath string
+	for _, folder := range strings.Split(p, "/") {
+		if folder == "" {
+			// Trailing / leading slash?
+			continue
+		}
+
+		currentPath = path.Join(currentPath, folder)
+		if err := fn(ctx, currentPath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}