diff --git a/.betterer.results b/.betterer.results index 10a8dd7e282..11c719f7a5f 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5866,8 +5866,10 @@ exports[`better eslint`] = { [0, 0, 0, "No untranslated strings in text props. Wrap text with or use t()", "7"], [0, 0, 0, "No untranslated strings in text props. Wrap text with or use t()", "8"], [0, 0, 0, "No untranslated strings in text props. Wrap text with or use t()", "9"], - [0, 0, 0, "No untranslated strings. Wrap text with ", "10"], - [0, 0, 0, "No untranslated strings. Wrap text with ", "11"] + [0, 0, 0, "No untranslated strings in text props. Wrap text with or use t()", "10"], + [0, 0, 0, "No untranslated strings in text props. Wrap text with or use t()", "11"], + [0, 0, 0, "No untranslated strings. Wrap text with ", "12"], + [0, 0, 0, "No untranslated strings. Wrap text with ", "13"] ], "public/app/features/provisioning/FileHistoryPage.tsx:5381": [ [0, 0, 0, "No untranslated strings. Wrap text with ", "0"], diff --git a/pkg/apis/provisioning/v0alpha1/jobs.go b/pkg/apis/provisioning/v0alpha1/jobs.go index 55e8bb4aa56..747907e62e5 100644 --- a/pkg/apis/provisioning/v0alpha1/jobs.go +++ b/pkg/apis/provisioning/v0alpha1/jobs.go @@ -97,11 +97,14 @@ type ExportJobOptions struct { // Preserve history (if possible) History bool `json:"history,omitempty"` - // Target branch for export + // Target branch for export (only git) Branch string `json:"branch,omitempty"` - // File prefix + // Target file prefix Prefix string `json:"prefix,omitempty"` + + // Include the identifier in the exported metadata + Identifier bool `json:"identifier"` } // The job status @@ -137,6 +140,8 @@ type JobResourceSummary struct { Create int64 `json:"create,omitempty"` Update int64 `json:"update,omitempty"` Delete int64 `json:"delete,omitempty"` + Write int64 `json:"write,omitempty"` // Create or update (export) + Error int64 `json:"error,omitempty"` // The error count // No action required (useful for sync) Noop int64 `json:"noop,omitempty"` diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go index a978dc83570..6341ab63f3a 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go @@ -109,19 +109,28 @@ func schema_pkg_apis_provisioning_v0alpha1_ExportJobOptions(ref common.Reference }, "branch": { SchemaProps: spec.SchemaProps{ - Description: "Target branch for export", + Description: "Target branch for export (only git)", Type: []string{"string"}, Format: "", }, }, "prefix": { SchemaProps: spec.SchemaProps{ - Description: "File prefix", + Description: "Target file prefix", Type: []string{"string"}, Format: "", }, }, + "identifier": { + SchemaProps: spec.SchemaProps{ + Description: "Include the identifier in the exported metadata", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, }, + Required: []string{"identifier"}, }, }, } @@ -576,6 +585,19 @@ func schema_pkg_apis_provisioning_v0alpha1_JobResourceSummary(ref common.Referen Format: "int64", }, }, + "write": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int64", + }, + }, + "error": { + SchemaProps: spec.SchemaProps{ + Description: "Create or update (export)", + Type: []string{"integer"}, + Format: "int64", + }, + }, "noop": { SchemaProps: spec.SchemaProps{ Description: "No action required (useful for sync)", diff --git a/pkg/registry/apis/provisioning/export.go b/pkg/registry/apis/provisioning/export.go index 14c3d7c3fbc..3215991ed8b 100644 --- a/pkg/registry/apis/provisioning/export.go +++ b/pkg/registry/apis/provisioning/export.go @@ -52,7 +52,7 @@ func (c *exportConnector) Connect( return nil, err } cfg := repo.Config() - if !cfg.Spec.ReadOnly { + if cfg.Spec.ReadOnly { return nil, &apierrors.StatusError{ErrStatus: v1.Status{ Code: http.StatusPreconditionFailed, Message: "Repository is read only", diff --git a/pkg/registry/apis/provisioning/jobs/exporter.go b/pkg/registry/apis/provisioning/jobs/exporter.go index 2565d2517f3..49dd350a10d 100644 --- a/pkg/registry/apis/provisioning/jobs/exporter.go +++ b/pkg/registry/apis/provisioning/jobs/exporter.go @@ -2,11 +2,10 @@ package jobs import ( "context" + "encoding/json" "errors" "fmt" - "path" - "gopkg.in/yaml.v3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -17,10 +16,13 @@ import ( apiutils "github.com/grafana/grafana/pkg/apimachinery/utils" folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/infra/slugify" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath" ) +// Export reads from grafana and writes to a a repository type Exporter interface { Export(ctx context.Context, repo repository.Repository, @@ -30,7 +32,7 @@ type Exporter interface { } type exporter struct { - client *resources.DynamicClient + client *resources.DynamicClient // namespaced! dashboards dynamic.ResourceInterface folders dynamic.ResourceInterface repository repository.Repository @@ -38,9 +40,11 @@ type exporter struct { func NewExporter( repo repository.Repository, - parser *resources.Parser, + dynamicClient *resources.DynamicClient, ) (Exporter, error) { - dynamicClient := parser.Client() + if dynamicClient.GetNamespace() != repo.Config().Namespace { + return nil, fmt.Errorf("bad setup, exporter needs a namespaced client matching the repository") + } folders := dynamicClient.Resource(schema.GroupVersionResource{ Group: folders.GROUP, Version: folders.VERSION, @@ -67,48 +71,60 @@ func (r *exporter) Export(ctx context.Context, progress func(provisioning.JobStatus) error, ) (*provisioning.JobStatus, error) { logger := logging.FromContext(ctx) - err := progress(provisioning.JobStatus{ + status := provisioning.JobStatus{ State: provisioning.JobStateWorking, - Message: "getting folder tree...", - }) + Message: "reading folder tree...", + } + err := progress(status) if err != nil { return nil, err } - var ref string - if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType { - ref = r.repository.Config().Spec.GitHub.Branch - } - ns := r.repository.Config().GetNamespace() - logger = logger.With("ref", ref, "namespace", ns) - - err = progress(provisioning.JobStatus{ - State: provisioning.JobStateWorking, - Message: "exporting folders...", - }) - if err != nil { - return nil, err + ref := options.Branch // only valid for git (defaults to the configured repo) + if options.Prefix != "" { + options.Prefix = safepath.Clean(options.Prefix) } // TODO: handle pagination - rawFolders, err := r.folders.List(ctx, metav1.ListOptions{}) + rawList, err := r.folders.List(ctx, metav1.ListOptions{Limit: 10000}) if err != nil { return nil, fmt.Errorf("failed to list folders: %w", err) } + if rawList.GetContinue() != "" { + return nil, fmt.Errorf("unable to list all folders in one request: %s", rawList.GetContinue()) + } - unprovisionedFolders := make([]unstructured.Unstructured, 0, len(rawFolders.Items)) - for _, f := range rawFolders.Items { + // filter out the folders we already own + rawFolders := make([]unstructured.Unstructured, 0, len(rawList.Items)) + for _, f := range rawList.Items { repoName := f.GetAnnotations()[apiutils.AnnoKeyRepoName] if repoName == repo.Config().GetName() { logger.Info("skip as folder is already in repository", "folder", f.GetName()) continue } - - unprovisionedFolders = append(unprovisionedFolders, f) + rawFolders = append(rawFolders, f) } - folderTree := resources.NewFolderTreeFromUnstructure(ctx, unprovisionedFolders) + status.Message = fmt.Sprintf("exporting folders (%d)...", len(rawFolders)) + err = progress(status) + if err != nil { + return nil, err + } + + folderTree := resources.NewFolderTreeFromUnstructure(ctx, rawFolders) + if options.Folder != "" { + return nil, fmt.Errorf("non-root folder not yet supported") + } + + // first create folders + summary := provisioning.JobResourceSummary{ + Group: folders.GROUP, + Resource: folders.RESOURCE, + } err = folderTree.Walk(ctx, func(ctx context.Context, folder resources.Folder) error { p := folder.Path + "/" + if options.Prefix != "" { + p = options.Prefix + "/" + p + } logger := logger.With("path", p) _, err = r.repository.Read(ctx, p, ref) @@ -117,34 +133,41 @@ func (r *exporter) Export(ctx context.Context, return fmt.Errorf("failed to check if folder exists before writing: %w", err) } else if err == nil { logger.Info("folder already exists") + summary.Noop++ return nil } - // ErrFileNotFound - if err := r.repository.Create(ctx, p, ref, nil, "export of folder `"+p+"` in namespace "+ns); err != nil { + // Create with an empty body will make a folder (or .keep file if unsupported) + if err := r.repository.Create(ctx, p, ref, nil, "export folder `"+p+"`"); err != nil { logger.Error("failed to write a folder in repository", "error", err) return fmt.Errorf("failed to write folder in repo: %w", err) } + summary.Create++ logger.Debug("successfully exported folder") return nil }) if err != nil { return nil, fmt.Errorf("failed to write folders: %w", err) } + status.Summary = append(status.Summary, summary) + status.Message = "writing dashboards..." - err = progress(provisioning.JobStatus{ - State: provisioning.JobStateWorking, - Message: "writing dashboards...", - }) + err = progress(status) if err != nil { return nil, err } + // TODO: handle pagination dashboardList, err := r.dashboards.List(ctx, metav1.ListOptions{Limit: 1000}) if err != nil { return nil, fmt.Errorf("failed to list dashboards: %w", err) } + commitMessage := fmt.Sprintf("grafana export to: %s", repo.Config().Name) + summary = provisioning.JobResourceSummary{ + Group: "dashboard.grafana.app", + Resource: "dashboards", + } for _, item := range dashboardList.Items { if ctx.Err() != nil { logger.Debug("cancelling replication process due to ctx error", "error", err) @@ -152,22 +175,17 @@ func (r *exporter) Export(ctx context.Context, } name := item.GetName() - logger := logger.With("item", name) - ns := r.repository.Config().GetNamespace() - if item.GetNamespace() != ns { - // This case shouldn't happen - logger.Error("skipping dashboard item due to mismatching namespace", "got", ns) - continue - } - repoName := item.GetAnnotations()[apiutils.AnnoKeyRepoName] if repoName == repo.Config().GetName() { - logger.Info("skip as dashboard is already in repository", "name", name) + logger.Info("skip dashboard since it is already in repository", "dashboard", name) continue } + title, _, _ := unstructured.NestedString(item.Object, "spec", "title") + if title == "" { + title = name + } folder := item.GetAnnotations()[apiutils.AnnoKeyFolder] - logger = logger.With("folder", folder) // Get the absolute path of the folder fid, ok := folderTree.DirPath(folder, "") @@ -177,30 +195,43 @@ func (r *exporter) Export(ctx context.Context, } delete(item.Object, "metadata") - marshalledBody, err := yaml.Marshal(item.Object) + if options.Identifier { + item.SetName(name) // keep the identifier in the metadata + } + + body, err := json.MarshalIndent(item.Object, "", " ") if err != nil { return nil, fmt.Errorf("failed to marshal dashboard %s: %w", name, err) } - fileName := path.Join(fid.Path, name+".yaml") - logger = logger.With("file", fileName) - _, err = r.repository.Read(ctx, fileName, ref) - if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) { - logger.Error("failed to check if file exists before writing", "error", err) - return nil, fmt.Errorf("failed to check if file exists before writing: %w", err) - } else if err != nil { // ErrFileNotFound - err = r.repository.Create(ctx, fileName, ref, marshalledBody, "export of dashboard "+name+" in namespace "+ns) - } else { - err = r.repository.Update(ctx, fileName, ref, marshalledBody, "export of dashboard "+name+" in namespace "+ns) + fileName := slugify.Slugify(title) + ".json" + if fid.Path != "" { + fileName, err = safepath.Join(fid.Path, fileName) + if err != nil { + return nil, fmt.Errorf("error adding file path %s: %w", title, err) + } } + if options.Prefix != "" { + fileName, err = safepath.Join(options.Prefix, fileName) + if err != nil { + return nil, fmt.Errorf("error adding path prefix %s: %w", options.Prefix, err) + } + } + + // Write the file + err = r.repository.Write(ctx, fileName, ref, body, commitMessage) if err != nil { + summary.Error++ logger.Error("failed to write a file in repository", "error", err) - return nil, fmt.Errorf("failed to write file in repo: %w", err) + if len(summary.Errors) < 20 { + summary.Errors = append(summary.Errors, fmt.Sprintf("error writing: %s", fileName)) + } + } else { + summary.Write++ } - logger.Debug("successfully exported item") } - - return &provisioning.JobStatus{ - State: provisioning.JobStateSuccess, - }, nil + status.Summary = append(status.Summary, summary) + status.State = provisioning.JobStateSuccess + status.Message = "" + return &status, nil } diff --git a/pkg/registry/apis/provisioning/jobs/worker.go b/pkg/registry/apis/provisioning/jobs/worker.go index e8e2734d741..5c0ac388bd4 100644 --- a/pkg/registry/apis/provisioning/jobs/worker.go +++ b/pkg/registry/apis/provisioning/jobs/worker.go @@ -144,7 +144,7 @@ func (g *JobWorker) Process(ctx context.Context, job provisioning.Job, progress if job.Spec.Export.Branch == "*dummy*" { exporter = &dummyExporter{} } else { - exporter, err = NewExporter(repo, parser) + exporter, err = NewExporter(repo, parser.Client()) if err != nil { return nil, err } diff --git a/pkg/registry/apis/provisioning/repository/github.go b/pkg/registry/apis/provisioning/repository/github.go index 729465cef7f..394b02b2348 100644 --- a/pkg/registry/apis/provisioning/repository/github.go +++ b/pkg/registry/apis/provisioning/repository/github.go @@ -285,6 +285,10 @@ func (r *githubRepository) Update(ctx context.Context, path, ref string, data [] return nil } +func (r *githubRepository) Write(ctx context.Context, path string, ref string, data []byte, message string) error { + return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message) +} + func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string) error { if ref == "" { ref = r.config.Spec.GitHub.Branch diff --git a/pkg/registry/apis/provisioning/repository/local.go b/pkg/registry/apis/provisioning/repository/local.go index 3837fd887f5..b3c5ffaa3d8 100644 --- a/pkg/registry/apis/provisioning/repository/local.go +++ b/pkg/registry/apis/provisioning/repository/local.go @@ -347,6 +347,23 @@ func (r *localRepository) Update(ctx context.Context, path string, ref string, d return os.WriteFile(path, data, 0600) } +func (r *localRepository) Write(ctx context.Context, path, ref string, data []byte, comment string) error { + if err := r.validateRequest(ref); err != nil { + return err + } + + path, err := safepath.Join(r.path, path) + if err != nil { + return fmt.Errorf("join path: %w", err) + } + + if strings.HasSuffix(path, "/") { + return os.MkdirAll(path, 0600) + } + + return os.WriteFile(path, data, 0600) +} + func (r *localRepository) Delete(ctx context.Context, path string, ref string, comment string) error { if err := r.validateRequest(ref); err != nil { return err diff --git a/pkg/registry/apis/provisioning/repository/repository.go b/pkg/registry/apis/provisioning/repository/repository.go index 309b1c8413a..86919f9ed28 100644 --- a/pkg/registry/apis/provisioning/repository/repository.go +++ b/pkg/registry/apis/provisioning/repository/repository.go @@ -2,9 +2,12 @@ package repository import ( "context" + "errors" + "fmt" "io/fs" "net/http" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -92,6 +95,10 @@ type Repository interface { // The data has already been validated and is ready for save Update(ctx context.Context, path, ref string, data []byte, message string) error + // Write a file to the repository. + // Functionally the same as Read then Create or Update, but more efficient depending on the backend + Write(ctx context.Context, path, ref string, data []byte, message string) error + // Delete a file in the remote repository Delete(ctx context.Context, path, ref, message string) error @@ -116,3 +123,14 @@ type VersionedRepository interface { LatestRef(ctx context.Context) (string, error) CompareFiles(ctx context.Context, base, ref string) ([]FileChange, error) } + +func writeWithReadThenCreateOrUpdate(ctx context.Context, r Repository, path, ref string, data []byte, comment string) error { + _, err := r.Read(ctx, path, ref) + if err != nil && !(errors.Is(err, ErrFileNotFound) || apierrors.IsNotFound(err)) { + return fmt.Errorf("failed to check if file exists before writing: %w", err) + } + if err == nil { + return r.Update(ctx, path, ref, data, comment) + } + return r.Create(ctx, path, ref, data, comment) +} diff --git a/pkg/registry/apis/provisioning/repository/s3.go b/pkg/registry/apis/provisioning/repository/s3.go index 1edce3f8404..7726c5b2b28 100644 --- a/pkg/registry/apis/provisioning/repository/s3.go +++ b/pkg/registry/apis/provisioning/repository/s3.go @@ -97,6 +97,10 @@ func (r *s3Repository) Delete(ctx context.Context, path string, ref string, comm } } +func (r *s3Repository) Write(ctx context.Context, path string, ref string, data []byte, message string) error { + return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message) +} + func (r *s3Repository) History(ctx context.Context, path string, ref string) ([]provisioning.HistoryItem, error) { return nil, &errors.StatusError{ ErrStatus: metav1.Status{ diff --git a/pkg/registry/apis/provisioning/repository/unknown.go b/pkg/registry/apis/provisioning/repository/unknown.go index 26419c6c0c1..827562b383d 100644 --- a/pkg/registry/apis/provisioning/repository/unknown.go +++ b/pkg/registry/apis/provisioning/repository/unknown.go @@ -84,6 +84,10 @@ func (r *unknownRepository) Update(ctx context.Context, path, ref string, data [ } } +func (r *unknownRepository) Write(ctx context.Context, path string, ref string, data []byte, message string) error { + return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message) +} + func (r *unknownRepository) Delete(ctx context.Context, path, ref, comment string) error { return &errors.StatusError{ ErrStatus: metav1.Status{ diff --git a/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json index 69510eedfcf..af79b991080 100644 --- a/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json +++ b/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json @@ -1250,9 +1250,12 @@ "application/json": { "schema": { "type": "object", + "required": [ + "identifier" + ], "properties": { "branch": { - "description": "Target branch for export", + "description": "Target branch for export (only git)", "type": "string" }, "folder": { @@ -1263,8 +1266,13 @@ "description": "Preserve history (if possible)", "type": "boolean" }, + "identifier": { + "description": "Include the identifier in the exported metadata", + "type": "boolean", + "default": false + }, "prefix": { - "description": "File prefix", + "description": "Target file prefix", "type": "string" } } @@ -1273,7 +1281,8 @@ "folder": "grafan-folder-ref", "history": true, "branch": "target-branch", - "prefix": "prefix/in/repo/tree" + "prefix": "prefix/in/repo/tree", + "identifier": false } } } @@ -2712,9 +2721,12 @@ }, "com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ExportJobOptions": { "type": "object", + "required": [ + "identifier" + ], "properties": { "branch": { - "description": "Target branch for export", + "description": "Target branch for export (only git)", "type": "string" }, "folder": { @@ -2725,8 +2737,13 @@ "description": "Preserve history (if possible)", "type": "boolean" }, + "identifier": { + "description": "Include the identifier in the exported metadata", + "type": "boolean", + "default": false + }, "prefix": { - "description": "File prefix", + "description": "Target file prefix", "type": "string" } } @@ -2891,6 +2908,11 @@ "type": "integer", "format": "int64" }, + "error": { + "description": "Create or update (export)", + "type": "integer", + "format": "int64" + }, "errors": { "description": "Report errors for this resource type This may not be an exhaustive list and recommend looking at the logs for more info", "type": "array", @@ -2913,6 +2935,10 @@ "update": { "type": "integer", "format": "int64" + }, + "write": { + "type": "integer", + "format": "int64" } } }, diff --git a/public/app/features/provisioning/ExportToRepository.tsx b/public/app/features/provisioning/ExportToRepository.tsx index e66e7560a23..046cc39e8dd 100644 --- a/public/app/features/provisioning/ExportToRepository.tsx +++ b/public/app/features/provisioning/ExportToRepository.tsx @@ -52,6 +52,10 @@ export function ExportToRepository({ repo }: Props) { + + + + diff --git a/public/app/features/provisioning/api/endpoints.gen.ts b/public/app/features/provisioning/api/endpoints.gen.ts index 3fd884a7068..456f6dd474f 100644 --- a/public/app/features/provisioning/api/endpoints.gen.ts +++ b/public/app/features/provisioning/api/endpoints.gen.ts @@ -401,13 +401,15 @@ export type CreateRepositoryExportArg = { /** name of the Job */ name: string; body: { - /** Target branch for export */ + /** Target branch for export (only git) */ branch?: string; /** The source folder (or empty) to export */ folder?: string; /** Preserve history (if possible) */ history?: boolean; - /** File prefix */ + /** Include the identifier in the exported metadata */ + identifier: boolean; + /** Target file prefix */ prefix?: string; }; }; @@ -659,13 +661,15 @@ export type ObjectMeta = { uid?: string; }; export type ExportJobOptions = { - /** Target branch for export */ + /** Target branch for export (only git) */ branch?: string; /** The source folder (or empty) to export */ folder?: string; /** Preserve history (if possible) */ history?: boolean; - /** File prefix */ + /** Include the identifier in the exported metadata */ + identifier: boolean; + /** Target file prefix */ prefix?: string; }; export type PullRequestJobOptions = { @@ -699,6 +703,8 @@ export type JobSpec = { export type JobResourceSummary = { create?: number; delete?: number; + /** Create or update (export) */ + error?: number; /** Report errors for this resource type This may not be an exhaustive list and recommend looking at the logs for more info */ errors?: string[]; group?: string; @@ -706,6 +712,7 @@ export type JobResourceSummary = { noop?: number; resource?: string; update?: number; + write?: number; }; export type JobStatus = { errors?: string[];