From ef300cc878d4b14c124ff10ddda14832f32ebf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Jim=C3=A9nez=20S=C3=A1nchez?= Date: Wed, 29 Jan 2025 18:42:08 +0100 Subject: [PATCH] [PROVISIONING] Support empty folders (#99560) * Support empty folders * Make it work * Add FIXMEs * Fix some issues * Move logic to tree * Further refactoring * Further refactoring * fix issue with root folder * Further refactoring * Fix bug with nested * Add Walk Function * Move walk to separate file * Add comment * Move walk to safepath * Fix bug for parent folder * Use the tree * Rename argument in walk function * WIP: walk over the tree * Filter out unprovisioned folders * Use the annotation instead * Fix lint issue * Implement basic tree walk * Use v1alpha1 for dashboards * Fix local provisioning * Fix folder creation local * Fix styling issue * Revert "Fix styling issue" This reverts commit b0f77a8f74e9f016c2860910be2c52bc2fa411c7. * Update pkg/registry/apis/provisioning/jobs/exporter.go Co-authored-by: Mariell Hoversholm --------- Co-authored-by: Ryan McKinley Co-authored-by: Mariell Hoversholm --- pkg/registry/apis/provisioning/files.go | 41 ++- .../apis/provisioning/jobs/exporter.go | 102 +++++-- pkg/registry/apis/provisioning/jobs/syncer.go | 258 +++++++++--------- .../apis/provisioning/repository/github.go | 46 +++- .../apis/provisioning/repository/local.go | 44 ++- .../apis/provisioning/resources/id.go | 10 + .../apis/provisioning/resources/tree.go | 83 ++++-- .../apis/provisioning/safepath/walk.go | 31 +++ 8 files changed, 434 insertions(+), 181 deletions(-) create mode 100644 pkg/registry/apis/provisioning/safepath/walk.go diff --git a/pkg/registry/apis/provisioning/files.go b/pkg/registry/apis/provisioning/files.go index 15a00c5a84a..1edde27db2f 100644 --- a/pkg/registry/apis/provisioning/files.go +++ b/pkg/registry/apis/provisioning/files.go @@ -8,6 +8,7 @@ import ( "net/http" "path" "strings" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,7 +74,9 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. } filePath := strings.TrimPrefix(r.URL.Path[idx+len(prefix):], "/") - if filePath == "" || strings.HasSuffix(filePath, "/") { + isFolderPath := strings.HasSuffix(filePath, "/") + + if r.Method == http.MethodGet && (filePath == "" || isFolderPath) { if len(filePath) > 0 { responder.Error(apierrors.NewBadRequest("folder navigation not yet supported")) return @@ -100,13 +103,15 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. return } - switch path.Ext(filePath) { - case ".json", ".yaml", ".yml": - // ok - default: - logger.Debug("got a file extension that was not JSON or YAML", "extension", path.Ext(filePath)) - responder.Error(apierrors.NewBadRequest("only yaml and json files supported")) - return + if !isFolderPath { + switch path.Ext(filePath) { + case ".json", ".yaml", ".yml": + // ok + default: + logger.Debug("got a file extension that was not JSON or YAML", "extension", path.Ext(filePath)) + responder.Error(apierrors.NewBadRequest("only yaml and json files supported")) + return + } } var obj *provisioning.ResourceWrapper @@ -117,7 +122,11 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. case http.MethodPost: obj, err = s.doWrite(ctx, false, repo, filePath, ref, message, r) case http.MethodPut: - obj, err = s.doWrite(ctx, true, repo, filePath, ref, message, r) + if isFolderPath { + err = apierrors.NewMethodNotSupported(provisioning.RepositoryResourceInfo.GroupResource(), r.Method) + } else { + obj, err = s.doWrite(ctx, true, repo, filePath, ref, message, r) + } case http.MethodDelete: obj, err = s.doDelete(ctx, repo, filePath, ref, message) default: @@ -182,6 +191,20 @@ func (s *filesConnector) doWrite(ctx context.Context, update bool, repo reposito } defer func() { _ = req.Body.Close() }() + if strings.HasSuffix(path, "/") { + if err := repo.Create(ctx, path, ref, nil, message); err != nil { + return nil, fmt.Errorf("failed to create folder: %w", err) + } + return &provisioning.ResourceWrapper{ + Path: path, + Ref: ref, + Timestamp: &metav1.Time{Time: time.Now()}, + // TODO: should we return something here? + // TypeMeta: metav1.TypeMeta{}, + // Resource: provisioning.ResourceObjects{}, + }, nil + } + data, err := io.ReadAll(req.Body) if err != nil { return nil, err diff --git a/pkg/registry/apis/provisioning/jobs/exporter.go b/pkg/registry/apis/provisioning/jobs/exporter.go index d4b7da55173..7824d734a8f 100644 --- a/pkg/registry/apis/provisioning/jobs/exporter.go +++ b/pkg/registry/apis/provisioning/jobs/exporter.go @@ -9,11 +9,13 @@ import ( "gopkg.in/yaml.v3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "github.com/grafana/grafana-app-sdk/logging" apiutils "github.com/grafana/grafana/pkg/apimachinery/utils" + folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" @@ -29,6 +31,7 @@ type Exporter interface { type exporter struct { client *resources.DynamicClient + dashboards dynamic.ResourceInterface folders dynamic.ResourceInterface repository repository.Repository } @@ -39,13 +42,21 @@ func NewExporter( ) (Exporter, error) { dynamicClient := parser.Client() folders := dynamicClient.Resource(schema.GroupVersionResource{ - Group: "folder.grafana.app", - Version: "v0alpha1", - Resource: "folders", + Group: folders.GROUP, + Version: folders.VERSION, + Resource: folders.RESOURCE, }) + + dashboards := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "dashboard.grafana.app", + Version: "v1alpha1", + Resource: "dashboards", + }) + return &exporter{ client: dynamicClient, folders: folders, + dashboards: dashboards, repository: repo, }, nil } @@ -56,12 +67,6 @@ func (r *exporter) Export(ctx context.Context, progress func(provisioning.JobStatus) error, ) (*provisioning.JobStatus, error) { logger := logging.FromContext(ctx) - dashboardIface := r.client.Resource(schema.GroupVersionResource{ - Group: "dashboard.grafana.app", - Version: "v2alpha1", - Resource: "dashboards", - }) - err := progress(provisioning.JobStatus{ State: provisioning.JobStateWorking, Message: "getting folder tree...", @@ -69,12 +74,64 @@ func (r *exporter) Export(ctx context.Context, if err != nil { return nil, err } + var ref string + if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType { + ref = r.repository.Config().Spec.GitHub.Branch + } + ns := r.repository.Config().GetNamespace() + logger = logger.With("ref", ref, "namespace", ns) - folders, err := resources.FetchRepoFolderTree(ctx, r.client) + err = progress(provisioning.JobStatus{ + State: provisioning.JobStateWorking, + Message: "exporting folders...", + }) + if err != nil { + return nil, err + } + + // TODO: handle pagination + rawFolders, err := r.folders.List(ctx, metav1.ListOptions{}) if err != nil { return nil, fmt.Errorf("failed to list folders: %w", err) } + var unprovisionedFolders []unstructured.Unstructured + for _, f := range rawFolders.Items { + repoName := f.GetAnnotations()[apiutils.AnnoKeyRepoName] + if repoName == repo.Config().GetName() { + logger.Info("skip as folder is already in repository", "folder", f.GetName()) + continue + } + + unprovisionedFolders = append(unprovisionedFolders, f) + } + + folderTree := resources.NewFolderTreeFromUnstructure(ctx, unprovisionedFolders) + err = folderTree.Walk(ctx, func(ctx context.Context, folder resources.Folder) error { + p := folder.Path + "/" + logger := logger.With("path", p) + + _, err = r.repository.Read(ctx, p, ref) + if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) { + logger.Error("failed to check if folder exists before writing", "error", err) + return fmt.Errorf("failed to check if folder exists before writing: %w", err) + } else if err == nil { + logger.Info("folder already exists") + return nil + } + + // ErrFileNotFound + if err := r.repository.Create(ctx, p, ref, nil, "export of folder `"+p+"` in namespace "+ns); err != nil { + logger.Error("failed to write a folder in repository", "error", err) + return fmt.Errorf("failed to write folder in repo: %w", err) + } + logger.Debug("successfully exported folder") + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to write folders: %w", err) + } + err = progress(provisioning.JobStatus{ State: provisioning.JobStateWorking, Message: "writing dashboards...", @@ -83,7 +140,7 @@ func (r *exporter) Export(ctx context.Context, return nil, err } // TODO: handle pagination - dashboardList, err := dashboardIface.List(ctx, metav1.ListOptions{Limit: 1000}) + dashboardList, err := r.dashboards.List(ctx, metav1.ListOptions{Limit: 1000}) if err != nil { return nil, fmt.Errorf("failed to list dashboards: %w", err) } @@ -98,16 +155,25 @@ func (r *exporter) Export(ctx context.Context, logger := logger.With("item", name) ns := r.repository.Config().GetNamespace() if item.GetNamespace() != ns { - logger.Debug("skipping dashboard item due to mismatching namespace", "got", ns) + // This case shouldn't happen + logger.Error("skipping dashboard item due to mismatching namespace", "got", ns) + continue + } + + repoName := item.GetAnnotations()[apiutils.AnnoKeyRepoName] + if repoName == repo.Config().GetName() { + logger.Info("skip as dashboard is already in repository", "name", name) continue } folder := item.GetAnnotations()[apiutils.AnnoKeyFolder] logger = logger.With("folder", folder) - fid, ok := folders.DirPath(folder, r.repository.Config().Spec.Folder) + + // Get the absolute path of the folder + fid, ok := folderTree.DirPath(folder, "") if !ok { - logger.Debug("folder of item was not in tree of repository") - continue + logger.Error("folder of item was not in tree of repository") + return nil, fmt.Errorf("folder of item was not in tree of repository") } delete(item.Object, "metadata") @@ -118,12 +184,6 @@ func (r *exporter) Export(ctx context.Context, fileName := path.Join(fid.Path, name+".yaml") logger = logger.With("file", fileName) - var ref string - if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType { - ref = r.repository.Config().Spec.GitHub.Branch - } - logger = logger.With("ref", ref) - _, err = r.repository.Read(ctx, fileName, ref) if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) { logger.Error("failed to check if file exists before writing", "error", err) diff --git a/pkg/registry/apis/provisioning/jobs/syncer.go b/pkg/registry/apis/provisioning/jobs/syncer.go index 34bf0e49a0e..a77755e405e 100644 --- a/pkg/registry/apis/provisioning/jobs/syncer.go +++ b/pkg/registry/apis/provisioning/jobs/syncer.go @@ -23,6 +23,7 @@ import ( provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath" ) type Syncer struct { @@ -61,8 +62,23 @@ func (r *Syncer) Sync(ctx context.Context, complete bool) (string, error) { lastCommit := cfg.Status.Sync.Hash versionedRepo, isVersioned := r.repository.(repository.VersionedRepository) - if err := r.ensureRepositoryFolderExists(ctx); err != nil { - return "", fmt.Errorf("ensure repository folder exists: %w", err) + // Ensure the configured folder exists and it's managed by the repository + if cfg.Spec.Folder != "" { + title := cfg.Spec.Title + if title == "" { + title = cfg.Spec.Folder + } + + folder := resources.Folder{ + Path: "", + ID: cfg.Spec.Folder, + Title: title, + } + + // If the folder already exists, the parent won't be changed + if err := r.ensureFolderExists(ctx, folder, ""); err != nil { + return "", fmt.Errorf("ensure repository folder exists: %w", err) + } } logger := logging.FromContext(ctx) @@ -181,28 +197,46 @@ func (r *Syncer) deleteListResource(ctx context.Context, item provisioning.Resou // replicateTree replicates all files in the repository. func (r *Syncer) replicateTree(ctx context.Context, ref string) error { - tree, err := r.repository.ReadTree(ctx, ref) + // Load the tree of the repository + fileTree, err := r.repository.ReadTree(ctx, ref) if err != nil { return fmt.Errorf("read tree: %w", err) } + // Load the list of resources in the cluster list, err := r.lister.List(ctx, r.client.GetNamespace(), r.repository.Config().Name) if err != nil { return fmt.Errorf("list resources: %w", err) } - if err := r.cleanUnnecessaryResources(ctx, tree, list); err != nil { + // Remove resources not longer present in the repository + if err := r.cleanUnnecessaryResources(ctx, fileTree, list); err != nil { return fmt.Errorf("clean up before replicating tree: %w", err) } + // Create folders + folderTree := resources.NewFolderTreeFromResourceList(list) + for _, entry := range fileTree { + parent := resources.ParentFolder(path.Dir(entry.Path), r.repository.Config()) + if err := r.ensureFolderPathExists(ctx, path.Dir(entry.Path), parent, folderTree); err != nil { + return fmt.Errorf("ensure folder path exists: %w", err) + } + } + + // Create a list of hashes to identify changed files hashes := make(map[string]string) for _, item := range list.Items { hashes[item.Path] = item.Hash } - // FIXME: Add support for empty folders - for _, entry := range tree { + // Create Dashboards + for _, entry := range fileTree { logger := logging.FromContext(ctx).With("file", entry.Path) + if hashes[entry.Path] == entry.Hash { + logger.Debug("file is up to date") + continue + } + if !entry.Blob { logger.Debug("ignoring non-blob entry") continue @@ -213,11 +247,6 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error { continue } - if hashes[entry.Path] == entry.Hash { - logger.Debug("file is up to date") - continue - } - info, err := r.repository.Read(ctx, entry.Path, ref) if err != nil { return fmt.Errorf("read file: %w", err) @@ -227,7 +256,12 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error { info.Hash = entry.Hash info.Modified = nil // modified? - if err := r.replicateFile(ctx, info); err != nil { + parsed, err := r.parseResource(ctx, info) + if err != nil { + return fmt.Errorf("parse resource: %w", err) + } + + if err := r.replicateFile(ctx, parsed, folderTree); err != nil { if errors.Is(err, resources.ErrUnableToReadResourceBytes) { logger.Info("file does not contain a resource") continue @@ -241,21 +275,16 @@ func (r *Syncer) replicateTree(ctx context.Context, ref string) error { // replicateFile creates a new resource in the cluster. // If the resource already exists, it will be updated. -func (r *Syncer) replicateFile(ctx context.Context, fileInfo *repository.FileInfo) error { - logger := logging.FromContext(ctx).With("file", fileInfo.Path, "ref", fileInfo.Ref) - file, err := r.parseResource(ctx, fileInfo) - if err != nil { - return err - } +func (r *Syncer) replicateFile(ctx context.Context, file *resources.ParsedResource, folderTree *resources.FolderTree) error { + logger := logging.FromContext(ctx).With("file", file.Info.Path, "ref", file.Info.Ref) logger = logger.With("action", file.Action, "name", file.Obj.GetName(), "file_namespace", file.Obj.GetNamespace(), "namespace", r.client.GetNamespace()) - parent, err := r.createFolderPath(ctx, fileInfo.Path) - if err != nil { - return fmt.Errorf("failed to create folder path: %w", err) - } + parent := resources.ParentFolder(file.Info.Path, r.repository.Config()) logger = logger.With("folder", parent) - if parent != "" { + if !folderTree.In(parent) { + return fmt.Errorf("failed to find parent in tree for %s in %s", file.Info.Path, parent) + } file.Meta.SetFolder(parent) } @@ -303,71 +332,24 @@ func (r *Syncer) replicateFile(ctx context.Context, fileInfo *repository.FileInf return nil } -func (r *Syncer) createFolderPath(ctx context.Context, filePath string) (string, error) { - dir := path.Dir(filePath) - parent := r.repository.Config().Spec.Folder - if dir == "." || dir == "/" { - return parent, nil - } - - logger := logging.FromContext(ctx).With("file", filePath) - - var currentPath string - for _, folder := range strings.Split(dir, "/") { - if folder == "" { - // Trailing / leading slash? - continue - } - - currentPath = path.Join(currentPath, folder) - folderID := resources.ParseFolder(currentPath, r.repository.Config().GetName()) - - logger := logger.With("folder", currentPath) - obj, err := r.folders.Get(ctx, folderID.ID, metav1.GetOptions{}) - // FIXME: Check for IsNotFound properly - if obj != nil || err == nil { - logger.Debug("folder already existed") - parent = folderID.ID - continue - } - - obj = &unstructured.Unstructured{ - Object: map[string]interface{}{ - "spec": map[string]any{ - "title": folderID.Title, - "description": "Repository-managed folder.", - }, - }, - } - - meta, err := utils.MetaAccessor(obj) - if err != nil { - return "", fmt.Errorf("create meta accessor for the object: %w", err) - } - - obj.SetNamespace(r.client.GetNamespace()) - obj.SetName(folderID.ID) - meta.SetFolder(parent) - meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ - Name: r.repository.Config().Name, - Path: currentPath, - Hash: "", // FIXME: which hash? - Timestamp: nil, // ???&info.Modified.Time, - }) - - _, err = r.folders.Create(ctx, obj, metav1.CreateOptions{}) - if err != nil { - return parent, fmt.Errorf("failed to create folder '%s': %w", folderID.ID, err) - } - - parent = folderID.ID - logger.Info("folder created") - } - - return parent, nil -} - func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.FileChange) error { + // Create an empty tree to avoid loading all folders unnecessarily + folderTree := resources.NewEmptyFolderTree() + + // Create folder structure first + for _, change := range changes { + if change.Action == repository.FileActionDeleted { + // FIXME: this will leave empty folder behind until the next sync + continue + } + + parent := resources.ParentFolder(path.Dir(change.Path), r.repository.Config()) + if err := r.ensureFolderPathExists(ctx, path.Dir(change.Path), parent, folderTree); err != nil { + return fmt.Errorf("ensure folder path exists: %w", err) + } + } + + // Replicate the file changes for _, change := range changes { if resources.ShouldIgnorePath(change.Path) { continue @@ -378,9 +360,14 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File return fmt.Errorf("read file: %w", err) } + parsed, err := r.parseResource(ctx, fileInfo) + if err != nil { + return fmt.Errorf("parse resource: %w", err) + } + switch change.Action { case repository.FileActionCreated, repository.FileActionUpdated: - if err := r.replicateFile(ctx, fileInfo); err != nil { + if err := r.replicateFile(ctx, parsed, folderTree); err != nil { return fmt.Errorf("replicate file: %w", err) } case repository.FileActionRenamed: @@ -389,15 +376,19 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File if err != nil { return fmt.Errorf("read previous path: %w", err) } - if err := r.deleteFile(ctx, oldPath); err != nil { + oldParsed, err := r.parseResource(ctx, oldPath) + if err != nil { + return fmt.Errorf("parse old resource: %w", err) + } + if err := r.deleteFile(ctx, oldParsed); err != nil { return fmt.Errorf("delete file: %w", err) } - if err := r.replicateFile(ctx, fileInfo); err != nil { + if err := r.replicateFile(ctx, parsed, folderTree); err != nil { return fmt.Errorf("replicate file in new path: %w", err) } case repository.FileActionDeleted: - if err := r.deleteFile(ctx, fileInfo); err != nil { + if err := r.deleteFile(ctx, parsed); err != nil { return fmt.Errorf("delete file: %w", err) } } @@ -406,13 +397,8 @@ func (r *Syncer) replicateChanges(ctx context.Context, changes []repository.File return nil } -func (r *Syncer) deleteFile(ctx context.Context, fileInfo *repository.FileInfo) error { - file, err := r.parseResource(ctx, fileInfo) - if err != nil { - return err - } - - _, err = file.Client.Get(ctx, file.Obj.GetName(), metav1.GetOptions{}) +func (r *Syncer) deleteFile(ctx context.Context, file *resources.ParsedResource) error { + _, err := file.Client.Get(ctx, file.Obj.GetName(), metav1.GetOptions{}) // FIXME: Remove the 'false &&' when .Get returns 404 on 404 instead of 500. Until then, this is a really ugly workaround. if false && err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to check if object already exists: %w", err) @@ -446,25 +432,59 @@ func (r *Syncer) parseResource(ctx context.Context, fileInfo *repository.FileInf return file, nil } -func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error { - if r.repository.Config().Spec.Folder == "" { - return nil - } +// ensureFolderPathExists creates the folder structure in the cluster. +func (r *Syncer) ensureFolderPathExists(ctx context.Context, dirPath, parent string, folderTree *resources.FolderTree) error { + return safepath.Walk(ctx, dirPath, func(ctx context.Context, path string) error { + fid := resources.ParseFolder(path, r.repository.Config().GetName()) + if folderTree.In(fid.ID) { + // already visited + parent = fid.ID + return nil + } - obj, err := r.folders.Get(ctx, r.repository.Config().Spec.Folder, metav1.GetOptions{}) + if err := r.ensureFolderExists(ctx, fid, parent); err != nil { + return fmt.Errorf("ensure folder exists: %w", err) + } + + folderTree.Add(fid, parent) + parent = fid.ID + + return nil + }) +} + +// ensureFolderExists creates the folder if it doesn't exist. +// If the folder already exists: +// - it will update the repository info. +// - it will keep the parent folder already set +func (r *Syncer) ensureFolderExists(ctx context.Context, folder resources.Folder, parent string) error { + cfg := r.repository.Config() + repoInfo := &utils.ResourceRepositoryInfo{ + Name: cfg.GetName(), + Path: folder.Path, + Hash: "", // FIXME: which hash? + Timestamp: nil, // ???&info.Modified.Time, + } + obj, err := r.folders.Get(ctx, folder.ID, metav1.GetOptions{}) if err == nil { meta, err := utils.MetaAccessor(obj) if err != nil { return fmt.Errorf("create meta accessor for the object: %w", err) } - meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ - Name: r.repository.Config().Name, - Path: "", - Hash: "", // FIXME: which hash? - Timestamp: nil, // ???&info.Modified.Time, - }) + existingInfo, err := meta.GetRepositoryInfo() + if err != nil { + return fmt.Errorf("get repository info: %w", err) + } + // Skip update if already present with right values + if existingInfo.Name == repoInfo.Name && existingInfo.Path == repoInfo.Path { + return nil + } + + meta.SetRepositoryInfo(repoInfo) + + // TODO: should we keep the parent? if _, err := r.folders.Update(ctx, obj, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("failed to add repo info to configured folder: %w", err) } @@ -474,16 +494,10 @@ func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error { return fmt.Errorf("failed to check if folder exists: %w", err) } - cfg := r.repository.Config() - title := cfg.Spec.Title - if title == "" { - title = cfg.Spec.Folder - } - obj = &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": map[string]any{ - "title": title, // TODO: how do we want to get this? + "title": folder.Title, }, }, } @@ -494,13 +508,11 @@ func (r *Syncer) ensureRepositoryFolderExists(ctx context.Context) error { } obj.SetNamespace(cfg.GetNamespace()) - obj.SetName(cfg.Spec.Folder) - meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ - Name: r.repository.Config().Name, - Path: "", - Hash: "", // FIXME: which hash? - Timestamp: nil, // ???&info.Modified.Time, - }) + obj.SetName(folder.ID) + if parent != "" { + meta.SetFolder(parent) + } + meta.SetRepositoryInfo(repoInfo) if _, err := r.folders.Create(ctx, obj, metav1.CreateOptions{}); err != nil { return fmt.Errorf("failed to create folder: %w", err) diff --git a/pkg/registry/apis/provisioning/repository/github.go b/pkg/registry/apis/provisioning/repository/github.go index efe4a39c789..b921d06fccf 100644 --- a/pkg/registry/apis/provisioning/repository/github.go +++ b/pkg/registry/apis/provisioning/repository/github.go @@ -160,7 +160,10 @@ func (r *githubRepository) Read(ctx context.Context, filePath, ref string) (*Fil return nil, fmt.Errorf("get contents: %w", err) } if dirContent != nil { - return nil, fmt.Errorf("input path was a directory") + return &FileInfo{ + Path: filePath, + Ref: ref, + }, nil } data, err := content.GetFileContent() @@ -224,6 +227,16 @@ func (r *githubRepository) Create(ctx context.Context, path, ref string, data [] owner := r.config.Spec.GitHub.Owner repo := r.config.Spec.GitHub.Repository + // Create .keep file if it is a directory + if strings.HasSuffix(path, "/") { + if data != nil { + return apierrors.NewBadRequest("data cannot be provided for a directory") + } + + path = strings.TrimSuffix(path, "/") + "/.keep" + data = []byte{} + } + err := r.gh.CreateFile(ctx, owner, repo, path, ref, comment, data) if errors.Is(err, pgh.ErrResourceAlreadyExists) { return &apierrors.StatusError{ @@ -262,6 +275,9 @@ func (r *githubRepository) Update(ctx context.Context, path, ref string, data [] return fmt.Errorf("get content before file update: %w", err) } + if file.IsDirectory() { + return apierrors.NewBadRequest("cannot update a directory") + } if err := r.gh.UpdateFile(ctx, owner, repo, path, ref, comment, file.GetSHA(), data); err != nil { return fmt.Errorf("update file: %w", err) @@ -279,10 +295,14 @@ func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string return fmt.Errorf("create branch on delete: %w", err) } + return r.deleteRecursively(ctx, path, ref, comment) +} + +func (r *githubRepository) deleteRecursively(ctx context.Context, path, ref, comment string) error { owner := r.config.Spec.GitHub.Owner repo := r.config.Spec.GitHub.Repository - file, _, err := r.gh.GetContents(ctx, owner, repo, path, ref) + file, contents, err := r.gh.GetContents(ctx, owner, repo, path, ref) if err != nil { if errors.Is(err, pgh.ErrResourceNotFound) { return &apierrors.StatusError{ @@ -295,7 +315,24 @@ func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string return fmt.Errorf("finding file to delete: %w", err) } - return r.gh.DeleteFile(ctx, owner, repo, path, ref, comment, file.GetSHA()) + if file != nil && !file.IsDirectory() { + return r.gh.DeleteFile(ctx, owner, repo, path, ref, comment, file.GetSHA()) + } + + for _, c := range contents { + if c.IsDirectory() { + if err := r.deleteRecursively(ctx, c.GetPath(), ref, comment); err != nil { + return fmt.Errorf("delete file recursive: %w", err) + } + continue + } + + if err := r.gh.DeleteFile(ctx, owner, repo, c.GetPath(), ref, comment, c.GetSHA()); err != nil { + return fmt.Errorf("delete file: %w", err) + } + } + + return nil } func (r *githubRepository) History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error) { @@ -473,6 +510,9 @@ func (r *githubRepository) parsePushEvent(event *github.PushEvent) (*provisionin Job: &provisioning.JobSpec{ Repository: r.Config().GetName(), Action: provisioning.JobActionSync, + Sync: &provisioning.SyncOptions{ + Complete: false, + }, }, }, nil } diff --git a/pkg/registry/apis/provisioning/repository/local.go b/pkg/registry/apis/provisioning/repository/local.go index f2fbdafd086..3837fd887f5 100644 --- a/pkg/registry/apis/provisioning/repository/local.go +++ b/pkg/registry/apis/provisioning/repository/local.go @@ -191,29 +191,39 @@ func (r *localRepository) Read(ctx context.Context, filePath string, ref string) filePath, err := safepath.Join(r.path, filePath) if err != nil { - return nil, err + return nil, fmt.Errorf("join path: %w", err) } info, err := os.Stat(filePath) if errors.Is(err, os.ErrNotExist) { return nil, ErrFileNotFound } else if err != nil { - return nil, err + return nil, fmt.Errorf("stat file: %w", err) + } + + path := strings.TrimPrefix(filePath, safepath.Clean(r.path)) + if info.IsDir() { + return &FileInfo{ + Path: path, + Modified: &metav1.Time{ + Time: info.ModTime(), + }, + }, nil } //nolint:gosec data, err := os.ReadFile(filePath) if err != nil { - return nil, err + return nil, fmt.Errorf("read file: %w", err) } hash, _, err := r.calculateFileHash(filePath) if err != nil { - return nil, err + return nil, fmt.Errorf("calculate hash of file: %w", err) } return &FileInfo{ - Path: strings.TrimPrefix(filePath, safepath.Clean(r.path)), + Path: path, Data: data, Hash: hash, Modified: &metav1.Time{ @@ -281,11 +291,13 @@ func (r *localRepository) Create(ctx context.Context, sanitisedPath string, ref return err } + // before sanitizing, check if the path is a directory + isDir := strings.HasSuffix(sanitisedPath, "/") inputUnsafePath := sanitisedPath // do NOT use in path operations, only for responses! sanitisedPath, err := safepath.Join(r.path, sanitisedPath) if err != nil { - return err + return fmt.Errorf("join path: %w", err) } _, err = os.Stat(sanitisedPath) @@ -296,8 +308,20 @@ func (r *localRepository) Create(ctx context.Context, sanitisedPath string, ref return apierrors.NewAlreadyExists(provisioning.RepositoryResourceInfo.GroupResource(), inputUnsafePath) } + if isDir { + if data != nil { + return apierrors.NewBadRequest("data cannot be provided for a directory") + } + + if err := os.MkdirAll(sanitisedPath, 0700); err != nil { + return apierrors.NewInternalError(fmt.Errorf("failed to create path: %w", err)) + } + + return nil + } + if err := os.MkdirAll(path.Dir(sanitisedPath), 0700); err != nil { - return apierrors.NewInternalError(fmt.Errorf("failed to create file: %w", err)) + return apierrors.NewInternalError(fmt.Errorf("failed to create path: %w", err)) } return os.WriteFile(sanitisedPath, data, 0600) @@ -310,7 +334,11 @@ func (r *localRepository) Update(ctx context.Context, path string, ref string, d path, err := safepath.Join(r.path, path) if err != nil { - return err + return fmt.Errorf("join path: %w", err) + } + + if strings.HasSuffix(path, "/") { + return apierrors.NewBadRequest("cannot update a directory") } if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { diff --git a/pkg/registry/apis/provisioning/resources/id.go b/pkg/registry/apis/provisioning/resources/id.go index cb2830e9a17..a3a866bdf7a 100644 --- a/pkg/registry/apis/provisioning/resources/id.go +++ b/pkg/registry/apis/provisioning/resources/id.go @@ -5,6 +5,8 @@ import ( "encoding/base64" "path" "strings" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" ) // sanitiseKubeName removes all characters that don't fulfil the DNS subdomain name rules: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names @@ -131,6 +133,14 @@ func ParseFolder(dirPath, repositoryName string) Folder { } } +func ParentFolder(filePath string, repository *provisioning.Repository) string { + parent := path.Dir(filePath) + if parent == "." || parent == "/" { + return repository.Spec.Folder + } + return ParseFolder(parent, repository.GetName()).ID +} + func folderTitleFromPath(cleanDirPath string) string { titleIdx := strings.LastIndex(cleanDirPath, "/") title := cleanDirPath diff --git a/pkg/registry/apis/provisioning/resources/tree.go b/pkg/registry/apis/provisioning/resources/tree.go index ca59ee1999e..3525b37b02d 100644 --- a/pkg/registry/apis/provisioning/resources/tree.go +++ b/pkg/registry/apis/provisioning/resources/tree.go @@ -3,11 +3,13 @@ package resources import ( "context" "path" + "sort" + "strings" apiutils "github.com/grafana/grafana/pkg/apimachinery/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" ) // FolderTree contains the entire set of folders (at a given snapshot in time) of the Grafana instance. @@ -60,22 +62,46 @@ func (t *FolderTree) DirPath(folder, baseFolder string) (fid Folder, ok bool) { return fid, ok } -func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTree, error) { - iface := client.Resource(schema.GroupVersionResource{ - Group: "folder.grafana.app", - Version: "v0alpha1", - Resource: "folders", - }) +func (t *FolderTree) Add(folder Folder, parent string) { + t.tree[folder.ID] = parent + t.folders[folder.ID] = folder +} - // TODO: handle pagination - rawFolders, err := iface.List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err +type WalkFunc func(ctx context.Context, folder Folder) error + +func (t *FolderTree) Walk(ctx context.Context, fn WalkFunc) error { + toWalk := make([]Folder, 0, len(t.folders)) + for _, folder := range t.folders { + folder, _ := t.DirPath(folder.ID, "") + toWalk = append(toWalk, folder) } - tree := make(map[string]string, len(rawFolders.Items)) - folders := make(map[string]Folder, len(rawFolders.Items)) - for _, rf := range rawFolders.Items { + // sort by depth of the paths + sort.Slice(toWalk, func(i, j int) bool { + return len(strings.Split(toWalk[i].Path, "/")) < len(strings.Split(toWalk[j].Path, "/")) + }) + + for _, folder := range toWalk { + if err := fn(ctx, folder); err != nil { + return err + } + } + + return nil +} + +func NewEmptyFolderTree() *FolderTree { + return &FolderTree{ + tree: make(map[string]string, 0), + folders: make(map[string]Folder, 0), + } +} + +func NewFolderTreeFromUnstructure(ctx context.Context, rawFolders []unstructured.Unstructured) *FolderTree { + tree := make(map[string]string, len(rawFolders)) + folders := make(map[string]Folder, len(rawFolders)) + + for _, rf := range rawFolders { name := rf.GetName() // TODO: Can I use MetaAccessor here? parent := rf.GetAnnotations()[apiutils.AnnoKeyFolder] @@ -84,7 +110,8 @@ func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTre id := Folder{ Title: name, ID: name, - Path: "", // We'll set this later in the DirPath function :) + // TODO: should not this be be the annotation itself? + Path: "", // We'll set this later in the DirPath function :) } if title, ok, _ := unstructured.NestedString(rf.Object, "spec", "title"); ok { // If the title doesn't exist (it should), we'll just use the K8s name. @@ -96,5 +123,27 @@ func FetchRepoFolderTree(ctx context.Context, client *DynamicClient) (*FolderTre return &FolderTree{ tree: tree, folders: folders, - }, nil + } +} + +func NewFolderTreeFromResourceList(resources *provisioning.ResourceList) *FolderTree { + tree := make(map[string]string, len(resources.Items)) + folderIDs := make(map[string]Folder, len(resources.Items)) + for _, rf := range resources.Items { + if rf.Group != folders.GROUP { + continue + } + + tree[rf.Name] = rf.Folder + folderIDs[rf.Name] = Folder{ + Title: rf.Title, + ID: rf.Name, + Path: rf.Path, + } + } + + return &FolderTree{ + tree, + folderIDs, + } } diff --git a/pkg/registry/apis/provisioning/safepath/walk.go b/pkg/registry/apis/provisioning/safepath/walk.go new file mode 100644 index 00000000000..7c6cfa96568 --- /dev/null +++ b/pkg/registry/apis/provisioning/safepath/walk.go @@ -0,0 +1,31 @@ +package safepath + +import ( + "context" + "path" + "strings" +) + +type WalkFunc func(ctx context.Context, path string) error + +// Walk walks the given folder path and calls the given function for each folder. +func Walk(ctx context.Context, p string, fn WalkFunc) error { + if p == "." || p == "/" { + return nil + } + + var currentPath string + for _, folder := range strings.Split(p, "/") { + if folder == "" { + // Trailing / leading slash? + continue + } + + currentPath = path.Join(currentPath, folder) + if err := fn(ctx, currentPath); err != nil { + return err + } + } + + return nil +}