Provisioning: Export with write (#99853)

.betterer.results

@@ -5866,8 +5866,10 @@ exports[`better eslint`] = {
       [0, 0, 0, "No untranslated strings in text props. Wrap text with <Trans /> or use t()", "7"],
       [0, 0, 0, "No untranslated strings in text props. Wrap text with <Trans /> or use t()", "8"],
       [0, 0, 0, "No untranslated strings in text props. Wrap text with <Trans /> or use t()", "9"],
-      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "10"],
-      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "11"]
+      [0, 0, 0, "No untranslated strings in text props. Wrap text with <Trans /> or use t()", "10"],
+      [0, 0, 0, "No untranslated strings in text props. Wrap text with <Trans /> or use t()", "11"],
+      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "12"],
+      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "13"]
     ],
     "public/app/features/provisioning/FileHistoryPage.tsx:5381": [
       [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"],

pkg/apis/provisioning/v0alpha1/jobs.go

@@ -97,11 +97,14 @@ type ExportJobOptions struct {
 	// Preserve history (if possible)
 	History bool `json:"history,omitempty"`
 
-	// Target branch for export
+	// Target branch for export (only git)
 	Branch string `json:"branch,omitempty"`
 
-	// File prefix
+	// Target file prefix
 	Prefix string `json:"prefix,omitempty"`
+
+	// Include the identifier in the exported metadata
+	Identifier bool `json:"identifier"`
 }
 
 // The job status
@@ -137,6 +140,8 @@ type JobResourceSummary struct {
 	Create int64 `json:"create,omitempty"`
 	Update int64 `json:"update,omitempty"`
 	Delete int64 `json:"delete,omitempty"`
+	Write  int64 `json:"write,omitempty"` // Create or update (export)
+	Error  int64 `json:"error,omitempty"` // The error count
 
 	// No action required (useful for sync)
 	Noop int64 `json:"noop,omitempty"`

pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go

@@ -109,19 +109,28 @@ func schema_pkg_apis_provisioning_v0alpha1_ExportJobOptions(ref common.Reference
 					},
 					"branch": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Target branch for export",
+							Description: "Target branch for export (only git)",
 							Type:        []string{"string"},
 							Format:      "",
 						},
 					},
 					"prefix": {
 						SchemaProps: spec.SchemaProps{
-							Description: "File prefix",
+							Description: "Target file prefix",
 							Type:        []string{"string"},
 							Format:      "",
 						},
 					},
+					"identifier": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Include the identifier in the exported metadata",
+							Default:     false,
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
+				Required: []string{"identifier"},
 			},
 		},
 	}
@@ -576,6 +585,19 @@ func schema_pkg_apis_provisioning_v0alpha1_JobResourceSummary(ref common.Referen
 							Format: "int64",
 						},
 					},
+					"write": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"integer"},
+							Format: "int64",
+						},
+					},
+					"error": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Create or update (export)",
+							Type:        []string{"integer"},
+							Format:      "int64",
+						},
+					},
 					"noop": {
 						SchemaProps: spec.SchemaProps{
 							Description: "No action required (useful for sync)",

pkg/registry/apis/provisioning/export.go

@@ -52,7 +52,7 @@ func (c *exportConnector) Connect(
 		return nil, err
 	}
 	cfg := repo.Config()
-	if !cfg.Spec.ReadOnly {
+	if cfg.Spec.ReadOnly {
 		return nil, &apierrors.StatusError{ErrStatus: v1.Status{
 			Code:    http.StatusPreconditionFailed,
 			Message: "Repository is read only",

pkg/registry/apis/provisioning/jobs/exporter.go

@@ -2,11 +2,10 @@ package jobs
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
-	"path"
 
-	"gopkg.in/yaml.v3"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -17,10 +16,13 @@ import (
 	apiutils "github.com/grafana/grafana/pkg/apimachinery/utils"
 	folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
+	"github.com/grafana/grafana/pkg/infra/slugify"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
 	"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
+	"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
 )
 
+// Export reads from grafana and writes to a a repository
 type Exporter interface {
 	Export(ctx context.Context,
 		repo repository.Repository,
@@ -30,7 +32,7 @@ type Exporter interface {
 }
 
 type exporter struct {
-	client     *resources.DynamicClient
+	client     *resources.DynamicClient // namespaced!
 	dashboards dynamic.ResourceInterface
 	folders    dynamic.ResourceInterface
 	repository repository.Repository
@@ -38,9 +40,11 @@ type exporter struct {
 
 func NewExporter(
 	repo repository.Repository,
-	parser *resources.Parser,
+	dynamicClient *resources.DynamicClient,
 ) (Exporter, error) {
-	dynamicClient := parser.Client()
+	if dynamicClient.GetNamespace() != repo.Config().Namespace {
+		return nil, fmt.Errorf("bad setup, exporter needs a namespaced client matching the repository")
+	}
 	folders := dynamicClient.Resource(schema.GroupVersionResource{
 		Group:    folders.GROUP,
 		Version:  folders.VERSION,
@@ -67,48 +71,60 @@ func (r *exporter) Export(ctx context.Context,
 	progress func(provisioning.JobStatus) error,
 ) (*provisioning.JobStatus, error) {
 	logger := logging.FromContext(ctx)
-	err := progress(provisioning.JobStatus{
+	status := provisioning.JobStatus{
 		State:   provisioning.JobStateWorking,
-		Message: "getting folder tree...",
-	})
+		Message: "reading folder tree...",
+	}
+	err := progress(status)
 	if err != nil {
 		return nil, err
 	}
-	var ref string
-	if r.repository.Config().Spec.Type == provisioning.GitHubRepositoryType {
-		ref = r.repository.Config().Spec.GitHub.Branch
-	}
-	ns := r.repository.Config().GetNamespace()
-	logger = logger.With("ref", ref, "namespace", ns)
-
-	err = progress(provisioning.JobStatus{
-		State:   provisioning.JobStateWorking,
-		Message: "exporting folders...",
-	})
-	if err != nil {
-		return nil, err
+	ref := options.Branch // only valid for git (defaults to the configured repo)
+	if options.Prefix != "" {
+		options.Prefix = safepath.Clean(options.Prefix)
 	}
 
 	// TODO: handle pagination
-	rawFolders, err := r.folders.List(ctx, metav1.ListOptions{})
+	rawList, err := r.folders.List(ctx, metav1.ListOptions{Limit: 10000})
 	if err != nil {
 		return nil, fmt.Errorf("failed to list folders: %w", err)
 	}
+	if rawList.GetContinue() != "" {
+		return nil, fmt.Errorf("unable to list all folders in one request: %s", rawList.GetContinue())
+	}
 
-	unprovisionedFolders := make([]unstructured.Unstructured, 0, len(rawFolders.Items))
-	for _, f := range rawFolders.Items {
+	// filter out the folders we already own
+	rawFolders := make([]unstructured.Unstructured, 0, len(rawList.Items))
+	for _, f := range rawList.Items {
 		repoName := f.GetAnnotations()[apiutils.AnnoKeyRepoName]
 		if repoName == repo.Config().GetName() {
 			logger.Info("skip as folder is already in repository", "folder", f.GetName())
 			continue
 		}
-
-		unprovisionedFolders = append(unprovisionedFolders, f)
+		rawFolders = append(rawFolders, f)
 	}
 
-	folderTree := resources.NewFolderTreeFromUnstructure(ctx, unprovisionedFolders)
+	status.Message = fmt.Sprintf("exporting folders (%d)...", len(rawFolders))
+	err = progress(status)
+	if err != nil {
+		return nil, err
+	}
+
+	folderTree := resources.NewFolderTreeFromUnstructure(ctx, rawFolders)
+	if options.Folder != "" {
+		return nil, fmt.Errorf("non-root folder not yet supported")
+	}
+
+	// first create folders
+	summary := provisioning.JobResourceSummary{
+		Group:    folders.GROUP,
+		Resource: folders.RESOURCE,
+	}
 	err = folderTree.Walk(ctx, func(ctx context.Context, folder resources.Folder) error {
 		p := folder.Path + "/"
+		if options.Prefix != "" {
+			p = options.Prefix + "/" + p
+		}
 		logger := logger.With("path", p)
 
 		_, err = r.repository.Read(ctx, p, ref)
@@ -117,34 +133,41 @@ func (r *exporter) Export(ctx context.Context,
 			return fmt.Errorf("failed to check if folder exists before writing: %w", err)
 		} else if err == nil {
 			logger.Info("folder already exists")
+			summary.Noop++
 			return nil
 		}
 
-		// ErrFileNotFound
-		if err := r.repository.Create(ctx, p, ref, nil, "export of folder `"+p+"` in namespace "+ns); err != nil {
+		// Create with an empty body will make a folder (or .keep file if unsupported)
+		if err := r.repository.Create(ctx, p, ref, nil, "export folder `"+p+"`"); err != nil {
 			logger.Error("failed to write a folder in repository", "error", err)
 			return fmt.Errorf("failed to write folder in repo: %w", err)
 		}
+		summary.Create++
 		logger.Debug("successfully exported folder")
 		return nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to write folders: %w", err)
 	}
+	status.Summary = append(status.Summary, summary)
+	status.Message = "writing dashboards..."
 
-	err = progress(provisioning.JobStatus{
-		State:   provisioning.JobStateWorking,
-		Message: "writing dashboards...",
-	})
+	err = progress(status)
 	if err != nil {
 		return nil, err
 	}
+
 	// TODO: handle pagination
 	dashboardList, err := r.dashboards.List(ctx, metav1.ListOptions{Limit: 1000})
 	if err != nil {
 		return nil, fmt.Errorf("failed to list dashboards: %w", err)
 	}
 
+	commitMessage := fmt.Sprintf("grafana export to: %s", repo.Config().Name)
+	summary = provisioning.JobResourceSummary{
+		Group:    "dashboard.grafana.app",
+		Resource: "dashboards",
+	}
 	for _, item := range dashboardList.Items {
 		if ctx.Err() != nil {
 			logger.Debug("cancelling replication process due to ctx error", "error", err)
@@ -152,22 +175,17 @@ func (r *exporter) Export(ctx context.Context,
 		}
 
 		name := item.GetName()
-		logger := logger.With("item", name)
-		ns := r.repository.Config().GetNamespace()
-		if item.GetNamespace() != ns {
-			// This case shouldn't happen
-			logger.Error("skipping dashboard item due to mismatching namespace", "got", ns)
-			continue
-		}
-
 		repoName := item.GetAnnotations()[apiutils.AnnoKeyRepoName]
 		if repoName == repo.Config().GetName() {
-			logger.Info("skip as dashboard is already in repository", "name", name)
+			logger.Info("skip dashboard since it is already in repository", "dashboard", name)
 			continue
 		}
 
+		title, _, _ := unstructured.NestedString(item.Object, "spec", "title")
+		if title == "" {
+			title = name
+		}
 		folder := item.GetAnnotations()[apiutils.AnnoKeyFolder]
-		logger = logger.With("folder", folder)
 
 		// Get the absolute path of the folder
 		fid, ok := folderTree.DirPath(folder, "")
@@ -177,30 +195,43 @@ func (r *exporter) Export(ctx context.Context,
 		}
 
 		delete(item.Object, "metadata")
-		marshalledBody, err := yaml.Marshal(item.Object)
+		if options.Identifier {
+			item.SetName(name) // keep the identifier in the metadata
+		}
+
+		body, err := json.MarshalIndent(item.Object, "", "  ")
 		if err != nil {
 			return nil, fmt.Errorf("failed to marshal dashboard %s: %w", name, err)
 		}
-		fileName := path.Join(fid.Path, name+".yaml")
-		logger = logger.With("file", fileName)
 
-		_, err = r.repository.Read(ctx, fileName, ref)
-		if err != nil && !(errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
-			logger.Error("failed to check if file exists before writing", "error", err)
-			return nil, fmt.Errorf("failed to check if file exists before writing: %w", err)
-		} else if err != nil { // ErrFileNotFound
-			err = r.repository.Create(ctx, fileName, ref, marshalledBody, "export of dashboard "+name+" in namespace "+ns)
-		} else {
-			err = r.repository.Update(ctx, fileName, ref, marshalledBody, "export of dashboard "+name+" in namespace "+ns)
+		fileName := slugify.Slugify(title) + ".json"
+		if fid.Path != "" {
+			fileName, err = safepath.Join(fid.Path, fileName)
+			if err != nil {
+				return nil, fmt.Errorf("error adding file path %s: %w", title, err)
+			}
 		}
+		if options.Prefix != "" {
+			fileName, err = safepath.Join(options.Prefix, fileName)
+			if err != nil {
+				return nil, fmt.Errorf("error adding path prefix %s: %w", options.Prefix, err)
+			}
+		}
+
+		// Write the file
+		err = r.repository.Write(ctx, fileName, ref, body, commitMessage)
 		if err != nil {
+			summary.Error++
 			logger.Error("failed to write a file in repository", "error", err)
-			return nil, fmt.Errorf("failed to write file in repo: %w", err)
+			if len(summary.Errors) < 20 {
+				summary.Errors = append(summary.Errors, fmt.Sprintf("error writing: %s", fileName))
+			}
+		} else {
+			summary.Write++
 		}
-		logger.Debug("successfully exported item")
 	}
-
-	return &provisioning.JobStatus{
-		State: provisioning.JobStateSuccess,
-	}, nil
+	status.Summary = append(status.Summary, summary)
+	status.State = provisioning.JobStateSuccess
+	status.Message = ""
+	return &status, nil
 }

pkg/registry/apis/provisioning/jobs/worker.go

@@ -144,7 +144,7 @@ func (g *JobWorker) Process(ctx context.Context, job provisioning.Job, progress
 		if job.Spec.Export.Branch == "*dummy*" {
 			exporter = &dummyExporter{}
 		} else {
-			exporter, err = NewExporter(repo, parser)
+			exporter, err = NewExporter(repo, parser.Client())
 			if err != nil {
 				return nil, err
 			}

pkg/registry/apis/provisioning/repository/github.go

@@ -285,6 +285,10 @@ func (r *githubRepository) Update(ctx context.Context, path, ref string, data []
 	return nil
 }
 
+func (r *githubRepository) Write(ctx context.Context, path string, ref string, data []byte, message string) error {
+	return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message)
+}
+
 func (r *githubRepository) Delete(ctx context.Context, path, ref, comment string) error {
 	if ref == "" {
 		ref = r.config.Spec.GitHub.Branch

pkg/registry/apis/provisioning/repository/local.go

@@ -347,6 +347,23 @@ func (r *localRepository) Update(ctx context.Context, path string, ref string, d
 	return os.WriteFile(path, data, 0600)
 }
 
+func (r *localRepository) Write(ctx context.Context, path, ref string, data []byte, comment string) error {
+	if err := r.validateRequest(ref); err != nil {
+		return err
+	}
+
+	path, err := safepath.Join(r.path, path)
+	if err != nil {
+		return fmt.Errorf("join path: %w", err)
+	}
+
+	if strings.HasSuffix(path, "/") {
+		return os.MkdirAll(path, 0600)
+	}
+
+	return os.WriteFile(path, data, 0600)
+}
+
 func (r *localRepository) Delete(ctx context.Context, path string, ref string, comment string) error {
 	if err := r.validateRequest(ref); err != nil {
 		return err

pkg/registry/apis/provisioning/repository/repository.go

@@ -2,9 +2,12 @@ package repository
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"io/fs"
 	"net/http"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
@@ -92,6 +95,10 @@ type Repository interface {
 	// The data has already been validated and is ready for save
 	Update(ctx context.Context, path, ref string, data []byte, message string) error
 
+	// Write a file to the repository.
+	// Functionally the same as Read then Create or Update, but more efficient depending on the backend
+	Write(ctx context.Context, path, ref string, data []byte, message string) error
+
 	// Delete a file in the remote repository
 	Delete(ctx context.Context, path, ref, message string) error
 
@@ -116,3 +123,14 @@ type VersionedRepository interface {
 	LatestRef(ctx context.Context) (string, error)
 	CompareFiles(ctx context.Context, base, ref string) ([]FileChange, error)
 }
+
+func writeWithReadThenCreateOrUpdate(ctx context.Context, r Repository, path, ref string, data []byte, comment string) error {
+	_, err := r.Read(ctx, path, ref)
+	if err != nil && !(errors.Is(err, ErrFileNotFound) || apierrors.IsNotFound(err)) {
+		return fmt.Errorf("failed to check if file exists before writing: %w", err)
+	}
+	if err == nil {
+		return r.Update(ctx, path, ref, data, comment)
+	}
+	return r.Create(ctx, path, ref, data, comment)
+}

pkg/registry/apis/provisioning/repository/s3.go

@@ -97,6 +97,10 @@ func (r *s3Repository) Delete(ctx context.Context, path string, ref string, comm
 	}
 }
 
+func (r *s3Repository) Write(ctx context.Context, path string, ref string, data []byte, message string) error {
+	return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message)
+}
+
 func (r *s3Repository) History(ctx context.Context, path string, ref string) ([]provisioning.HistoryItem, error) {
 	return nil, &errors.StatusError{
 		ErrStatus: metav1.Status{

pkg/registry/apis/provisioning/repository/unknown.go

@@ -84,6 +84,10 @@ func (r *unknownRepository) Update(ctx context.Context, path, ref string, data [
 	}
 }
 
+func (r *unknownRepository) Write(ctx context.Context, path string, ref string, data []byte, message string) error {
+	return writeWithReadThenCreateOrUpdate(ctx, r, path, ref, data, message)
+}
+
 func (r *unknownRepository) Delete(ctx context.Context, path, ref, comment string) error {
 	return &errors.StatusError{
 		ErrStatus: metav1.Status{

pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json

@@ -1250,9 +1250,12 @@
             "application/json": {
               "schema": {
                 "type": "object",
+                "required": [
+                  "identifier"
+                ],
                 "properties": {
                   "branch": {
-                    "description": "Target branch for export",
+                    "description": "Target branch for export (only git)",
                     "type": "string"
                   },
                   "folder": {
@@ -1263,8 +1266,13 @@
                     "description": "Preserve history (if possible)",
                     "type": "boolean"
                   },
+                  "identifier": {
+                    "description": "Include the identifier in the exported metadata",
+                    "type": "boolean",
+                    "default": false
+                  },
                   "prefix": {
-                    "description": "File prefix",
+                    "description": "Target file prefix",
                     "type": "string"
                   }
                 }
@@ -1273,7 +1281,8 @@
                 "folder": "grafan-folder-ref",
                 "history": true,
                 "branch": "target-branch",
-                "prefix": "prefix/in/repo/tree"
+                "prefix": "prefix/in/repo/tree",
+                "identifier": false
               }
             }
           }
@@ -2712,9 +2721,12 @@
       },
       "com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ExportJobOptions": {
         "type": "object",
+        "required": [
+          "identifier"
+        ],
         "properties": {
           "branch": {
-            "description": "Target branch for export",
+            "description": "Target branch for export (only git)",
             "type": "string"
           },
           "folder": {
@@ -2725,8 +2737,13 @@
             "description": "Preserve history (if possible)",
             "type": "boolean"
           },
+          "identifier": {
+            "description": "Include the identifier in the exported metadata",
+            "type": "boolean",
+            "default": false
+          },
           "prefix": {
-            "description": "File prefix",
+            "description": "Target file prefix",
             "type": "string"
           }
         }
@@ -2891,6 +2908,11 @@
             "type": "integer",
             "format": "int64"
           },
+          "error": {
+            "description": "Create or update (export)",
+            "type": "integer",
+            "format": "int64"
+          },
           "errors": {
             "description": "Report errors for this resource type This may not be an exhaustive list and recommend looking at the logs for more info",
             "type": "array",
@@ -2913,6 +2935,10 @@
           "update": {
             "type": "integer",
             "format": "int64"
+          },
+          "write": {
+            "type": "integer",
+            "format": "int64"
           }
         }
       },

public/app/features/provisioning/ExportToRepository.tsx

@@ -52,6 +52,10 @@ export function ExportToRepository({ repo }: Props) {
             <Input placeholder="Prefix in the remote system" {...register('prefix')} />
           </Field>
 
+          <Field label="Identifier" description="Include the current identifier in exported metadata">
+            <Switch {...register('identifier')} />
+          </Field>
+
           <Field label="History" description="Include commits for each historical value">
             <Switch {...register('history')} />
           </Field>

public/app/features/provisioning/api/endpoints.gen.ts

@@ -401,13 +401,15 @@ export type CreateRepositoryExportArg = {
   /** name of the Job */
   name: string;
   body: {
-    /** Target branch for export */
+    /** Target branch for export (only git) */
     branch?: string;
     /** The source folder (or empty) to export */
     folder?: string;
     /** Preserve history (if possible) */
     history?: boolean;
-    /** File prefix */
+    /** Include the identifier in the exported metadata */
+    identifier: boolean;
+    /** Target file prefix */
     prefix?: string;
   };
 };
@@ -659,13 +661,15 @@ export type ObjectMeta = {
   uid?: string;
 };
 export type ExportJobOptions = {
-  /** Target branch for export */
+  /** Target branch for export (only git) */
   branch?: string;
   /** The source folder (or empty) to export */
   folder?: string;
   /** Preserve history (if possible) */
   history?: boolean;
-  /** File prefix */
+  /** Include the identifier in the exported metadata */
+  identifier: boolean;
+  /** Target file prefix */
   prefix?: string;
 };
 export type PullRequestJobOptions = {
@@ -699,6 +703,8 @@ export type JobSpec = {
 export type JobResourceSummary = {
   create?: number;
   delete?: number;
+  /** Create or update (export) */
+  error?: number;
   /** Report errors for this resource type This may not be an exhaustive list and recommend looking at the logs for more info */
   errors?: string[];
   group?: string;
@@ -706,6 +712,7 @@ export type JobResourceSummary = {
   noop?: number;
   resource?: string;
   update?: number;
+  write?: number;
 };
 export type JobStatus = {
   errors?: string[];