From cb18f50de50bc754487d8fdc27800c82f0b8b517 Mon Sep 17 00:00:00 2001 From: Roberto Jimenez Sanchez Date: Tue, 2 Dec 2025 17:24:27 +0100 Subject: [PATCH] Implement bulk export/push with resource list - Add Resources field to ExportJobOptions to support exporting specific resources - Implement ExportSpecificResources function that: - Validates resources (rejects folders, managed resources, unsupported types) - Loads unmanaged folder tree to replicate folder structure - Supports dashboard version conversion using shared shim logic - Replicates folder structure by concatenating Path + folder path from unmanaged tree - Update ExportWorker to dispatch to ExportSpecificResources when Resources list is provided - Add validation in validator.go for ExportJobOptions Resources field - Add comprehensive unit tests covering all scenarios - Update WriteResourceFileFromObject to handle folder path resolution --- .../pkg/apis/provisioning/v0alpha1/jobs.go | 6 + apps/provisioning/pkg/jobs/validator.go | 35 ++ .../provisioning/jobs/export/resources.go | 302 ++++++++-- .../jobs/export/resources_specific_test.go | 541 ++++++++++++++++++ .../apis/provisioning/jobs/export/worker.go | 8 + .../apis/provisioning/resources/resources.go | 16 +- 6 files changed, 862 insertions(+), 46 deletions(-) create mode 100644 pkg/registry/apis/provisioning/jobs/export/resources_specific_test.go diff --git a/apps/provisioning/pkg/apis/provisioning/v0alpha1/jobs.go b/apps/provisioning/pkg/apis/provisioning/v0alpha1/jobs.go index b96fb1a6d27..10596e6af98 100644 --- a/apps/provisioning/pkg/apis/provisioning/v0alpha1/jobs.go +++ b/apps/provisioning/pkg/apis/provisioning/v0alpha1/jobs.go @@ -133,6 +133,12 @@ type ExportJobOptions struct { // FIXME: we should validate this in admission hooks // Prefix in target file system Path string `json:"path,omitempty"` + + // Resources to export + // This option has been created because currently the frontend does not use + // standarized app platform APIs. For performance and API consistency reasons, the preferred option + // is it to use the resources. + Resources []ResourceRef `json:"resources,omitempty"` } type MigrateJobOptions struct { diff --git a/apps/provisioning/pkg/jobs/validator.go b/apps/provisioning/pkg/jobs/validator.go index 77232490b66..7ce7664e856 100644 --- a/apps/provisioning/pkg/jobs/validator.go +++ b/apps/provisioning/pkg/jobs/validator.go @@ -7,6 +7,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository/git" "github.com/grafana/grafana/apps/provisioning/pkg/safepath" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) // ValidateJob performs validation on the Job specification and returns an error if validation fails @@ -99,6 +100,40 @@ func validateExportJobOptions(opts *provisioning.ExportJobOptions) field.ErrorLi } } + // Validate resources if specified + if len(opts.Resources) > 0 { + for i, r := range opts.Resources { + resourcePath := field.NewPath("spec", "push", "resources").Index(i) + + // Validate required fields + if r.Name == "" { + list = append(list, field.Required(resourcePath.Child("name"), "resource name is required")) + } + if r.Kind == "" { + list = append(list, field.Required(resourcePath.Child("kind"), "resource kind is required")) + } + if r.Group == "" { + list = append(list, field.Required(resourcePath.Child("group"), "resource group is required")) + } + + // Validate that folders are not allowed + if r.Kind == resources.FolderKind.Kind || r.Group == resources.FolderResource.Group { + list = append(list, field.Invalid(resourcePath, r, "folders are not supported for export")) + continue // Skip further validation for folders + } + + // Validate that only supported resources are allowed + // Currently only Dashboard resources are supported (folders are rejected above) + if r.Kind != "" && r.Group != "" { + // Check if it's a Dashboard resource + isDashboard := r.Group == resources.DashboardResource.Group && r.Kind == "Dashboard" + if !isDashboard { + list = append(list, field.Invalid(resourcePath, r, "resource type is not supported for export")) + } + } + } + } + return list } diff --git a/pkg/registry/apis/provisioning/jobs/export/resources.go b/pkg/registry/apis/provisioning/jobs/export/resources.go index f1b8cd078de..d68d1f6b3be 100644 --- a/pkg/registry/apis/provisioning/jobs/export/resources.go +++ b/pkg/registry/apis/provisioning/jobs/export/resources.go @@ -13,6 +13,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/apps/provisioning/pkg/safepath" "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" @@ -23,6 +24,53 @@ import ( // The response status indicates the original stored version, so we can then request it in an un-converted form type conversionShim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) +// createDashboardConversionShim creates a conversion shim for dashboards that preserves the original API version. +// It caches version clients to avoid repeated lookups. +func createDashboardConversionShim(ctx context.Context, clients resources.ResourceClients, gvr schema.GroupVersionResource) (conversionShim, map[string]dynamic.ResourceInterface) { + versionClients := make(map[string]dynamic.ResourceInterface) + shim := func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) { + // Check if there's a stored version in the conversion status. + // This indicates the original API version the dashboard was created with, + // which should be preserved during export regardless of whether conversion succeeded or failed. + storedVersion, _, _ := unstructured.NestedString(item.Object, "status", "conversion", "storedVersion") + if storedVersion != "" { + // For v0 we can simply fallback -- the full model is saved + if strings.HasPrefix(storedVersion, "v0") { + item.SetAPIVersion(fmt.Sprintf("%s/%s", gvr.Group, storedVersion)) + return item, nil + } + + // For any other version (v1, v2, v3, etc.), fetch the original version via client + // Check if we already have a client cached for this version + versionClient, ok := versionClients[storedVersion] + if !ok { + // Dynamically construct the GroupVersionResource for any version + versionGVR := schema.GroupVersionResource{ + Group: gvr.Group, + Version: storedVersion, + Resource: gvr.Resource, + } + var err error + versionClient, _, err = clients.ForResource(ctx, versionGVR) + if err != nil { + return nil, fmt.Errorf("get client for version %s: %w", storedVersion, err) + } + versionClients[storedVersion] = versionClient + } + return versionClient.Get(ctx, item.GetName(), metav1.GetOptions{}) + } + + // If conversion failed but there's no storedVersion, this is an error condition + failed, _, _ := unstructured.NestedBool(item.Object, "status", "conversion", "failed") + if failed { + return nil, fmt.Errorf("conversion failed but no storedVersion available") + } + + return item, nil + } + return shim, versionClients +} + func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { progress.SetMessage(ctx, "start resource export") for _, kind := range resources.SupportedProvisioningResources { @@ -40,48 +88,7 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, // When requesting dashboards over the v1 api, we want to keep the original apiVersion if conversion fails var shim conversionShim if kind.GroupResource() == resources.DashboardResource.GroupResource() { - // Cache clients for different versions - versionClients := make(map[string]dynamic.ResourceInterface) - shim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) { - // Check if there's a stored version in the conversion status. - // This indicates the original API version the dashboard was created with, - // which should be preserved during export regardless of whether conversion succeeded or failed. - storedVersion, _, _ := unstructured.NestedString(item.Object, "status", "conversion", "storedVersion") - if storedVersion != "" { - // For v0 we can simply fallback -- the full model is saved - if strings.HasPrefix(storedVersion, "v0") { - item.SetAPIVersion(fmt.Sprintf("%s/%s", kind.Group, storedVersion)) - return item, nil - } - - // For any other version (v1, v2, v3, etc.), fetch the original version via client - // Check if we already have a client cached for this version - versionClient, ok := versionClients[storedVersion] - if !ok { - // Dynamically construct the GroupVersionResource for any version - versionGVR := schema.GroupVersionResource{ - Group: kind.Group, - Version: storedVersion, - Resource: kind.Resource, - } - var err error - versionClient, _, err = clients.ForResource(ctx, versionGVR) - if err != nil { - return nil, fmt.Errorf("get client for version %s: %w", storedVersion, err) - } - versionClients[storedVersion] = versionClient - } - return versionClient.Get(ctx, item.GetName(), metav1.GetOptions{}) - } - - // If conversion failed but there's no storedVersion, this is an error condition - failed, _, _ := unstructured.NestedBool(item.Object, "status", "conversion", "failed") - if failed { - return nil, fmt.Errorf("conversion failed but no storedVersion available") - } - - return item, nil - } + shim, _ = createDashboardConversionShim(ctx, clients, kind) } if err := exportResource(ctx, kind.Resource, options, client, shim, repositoryResources, progress); err != nil { @@ -92,6 +99,217 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, return nil } +// ExportSpecificResources exports a list of specific resources identified by ResourceRef entries. +// It validates that resources are not folders, are supported, and are unmanaged. +func ExportSpecificResources(ctx context.Context, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + if len(options.Resources) == 0 { + return errors.New("no resources specified for export") + } + + progress.SetMessage(ctx, "exporting specific resources") + + // Load folder tree into memory so we can resolve folder paths for resources + // This is needed to replicate the folder structure when exporting + progress.SetMessage(ctx, "loading folder tree from API server") + folderClient, err := clients.Folder(ctx) + if err != nil { + return fmt.Errorf("get folder client: %w", err) + } + + tree := resources.NewEmptyFolderTree() + if err := resources.ForEach(ctx, folderClient, func(item *unstructured.Unstructured) error { + if tree.Count() >= resources.MaxNumberOfFolders { + return errors.New("too many folders") + } + meta, err := utils.MetaAccessor(item) + if err != nil { + return fmt.Errorf("extract meta accessor: %w", err) + } + + manager, _ := meta.GetManagerProperties() + // Skip if already managed by any manager (repository, file provisioning, etc.) + if manager.Identity != "" { + return nil + } + + return tree.AddUnstructured(item) + }); err != nil { + return fmt.Errorf("load folder tree: %w", err) + } + + // Create a shared dashboard conversion shim and cache for all dashboard resources + var dashboardShim conversionShim + + for _, resourceRef := range options.Resources { + result := jobs.JobResourceResult{ + Name: resourceRef.Name, + Group: resourceRef.Group, + Kind: resourceRef.Kind, + Action: repository.FileActionCreated, + } + + gvk := schema.GroupVersionKind{ + Group: resourceRef.Group, + Kind: resourceRef.Kind, + // Version is left empty so ForKind will use the preferred version + } + + // Validate: reject folders + if gvk.Kind == resources.FolderKind.Kind || gvk.Group == resources.FolderResource.Group { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("folders are not supported for export") + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + // Get client for this resource type + progress.SetMessage(ctx, fmt.Sprintf("Fetching resource %s/%s/%s", resourceRef.Group, resourceRef.Kind, resourceRef.Name)) + client, gvr, err := clients.ForKind(ctx, gvk) + if err != nil { + result.Error = fmt.Errorf("get client for %s/%s/%s: %w", resourceRef.Group, resourceRef.Kind, resourceRef.Name, err) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + // Validate: check if resource is supported + isSupported := false + for _, supported := range resources.SupportedProvisioningResources { + if supported.Group == gvr.Group && supported.Resource == gvr.Resource { + isSupported = true + break + } + } + if !isSupported { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("resource type %s/%s is not supported for export", gvr.Group, gvr.Resource) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + // Fetch the resource from the API server + item, err := client.Get(ctx, resourceRef.Name, metav1.GetOptions{}) + if err != nil { + result.Error = fmt.Errorf("get resource %s/%s/%s: %w", resourceRef.Group, resourceRef.Kind, resourceRef.Name, err) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + // Validate: check if resource is managed + meta, err := utils.MetaAccessor(item) + if err != nil { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("extracting meta accessor for resource %s: %w", result.Name, err) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + manager, _ := meta.GetManagerProperties() + // Reject if already managed by any manager (repository, file provisioning, etc.) + if manager.Identity != "" { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("resource %s/%s/%s is managed and cannot be exported", resourceRef.Group, resourceRef.Kind, resourceRef.Name) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + + // Handle dashboard version conversion using the shared shim logic + if gvr.GroupResource() == resources.DashboardResource.GroupResource() { + // Create or reuse the dashboard shim (shared across all dashboard resources) + if dashboardShim == nil { + dashboardShim, _ = createDashboardConversionShim(ctx, clients, gvr) + } + + item, err = dashboardShim(ctx, item) + if err != nil { + result.Error = fmt.Errorf("converting dashboard %s/%s/%s: %w", resourceRef.Group, resourceRef.Kind, resourceRef.Name, err) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + // Re-extract meta after shim conversion in case the item changed + meta, err = utils.MetaAccessor(item) + if err != nil { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("extracting meta accessor after conversion for resource %s: %w", result.Name, err) + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + continue + } + } + + // Get the folder path from the unmanaged folder tree and concatenate with Path + // This gives us the path in the unmanaged tree structure + exportPath := options.Path + resourceFolder := meta.GetFolder() + if resourceFolder != "" { + // Get the folder path from the unmanaged tree (rootFolder is empty string for unmanaged tree) + fid, ok := tree.DirPath(resourceFolder, "") + if ok && fid.Path != "" { + if exportPath != "" { + exportPath = safepath.Join(exportPath, fid.Path) + } else { + exportPath = fid.Path + } + } + } + + // Temporarily clear folder metadata so WriteResourceFileFromObject doesn't try to resolve + // folder paths from repository tree (we've already computed the path from unmanaged tree) + originalFolder := resourceFolder + if resourceFolder != "" { + meta.SetFolder("") + } + + // Export the resource + progress.SetMessage(ctx, fmt.Sprintf("Exporting resource %s/%s/%s", resourceRef.Group, resourceRef.Kind, resourceRef.Name)) + result.Path, err = repositoryResources.WriteResourceFileFromObject(ctx, item, resources.WriteOptions{ + Path: exportPath, // Path already includes folder structure from unmanaged tree + Ref: options.Branch, + }) + + // Restore original folder metadata + if originalFolder != "" { + meta.SetFolder(originalFolder) + } + + if errors.Is(err, resources.ErrAlreadyInRepository) { + result.Action = repository.FileActionIgnored + } else if err != nil { + result.Action = repository.FileActionIgnored + result.Error = fmt.Errorf("writing resource file for %s: %w", result.Name, err) + } + + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + } + + return nil +} + func exportResource(ctx context.Context, resource string, options provisioning.ExportJobOptions, diff --git a/pkg/registry/apis/provisioning/jobs/export/resources_specific_test.go b/pkg/registry/apis/provisioning/jobs/export/resources_specific_test.go new file mode 100644 index 00000000000..71dd4416e1e --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/resources_specific_test.go @@ -0,0 +1,541 @@ +package export + +import ( + "context" + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/apimachinery/utils" + mock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + provisioningV0 "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" +) + +// Helper function to create folder objects +func createFolderObject(name, parentFolder string) unstructured.Unstructured { + folder := unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resources.FolderResource.GroupVersion().String(), + "kind": "Folder", + "metadata": map[string]interface{}{ + "name": name, + }, + "spec": map[string]interface{}{ + "title": name, + }, + }, + } + if parentFolder != "" { + meta, _ := utils.MetaAccessor(&folder) + meta.SetFolder(parentFolder) + } + return folder +} + +// Helper function to create dashboard objects with folder +func createDashboardObjectWithFolder(name, folderID string) unstructured.Unstructured { + dashboard := createDashboardObject(name) + if folderID != "" { + meta, _ := utils.MetaAccessor(&dashboard) + meta.SetFolder(folderID) + } + return dashboard +} + +// Helper function to run ExportSpecificResources test +func runExportSpecificResourcesTest(t *testing.T, resourceRefs []provisioningV0.ResourceRef, folderItems []unstructured.Unstructured, setupProgress func(*jobs.MockJobProgressRecorder), setupResources func(*resources.MockRepositoryResources, *resources.MockResourceClients)) error { + resourceClients := resources.NewMockResourceClients(t) + mockProgress := jobs.NewMockJobProgressRecorder(t) + setupProgress(mockProgress) + + repoResources := resources.NewMockRepositoryResources(t) + setupResources(repoResources, resourceClients) + + options := provisioningV0.ExportJobOptions{ + Path: "grafana", + Branch: "feature/branch", + Resources: resourceRefs, + } + + err := ExportSpecificResources(context.Background(), options, resourceClients, repoResources, mockProgress) + + mockProgress.AssertExpectations(t) + repoResources.AssertExpectations(t) + resourceClients.AssertExpectations(t) + + return err +} + +func TestExportSpecificResources_Success(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "dashboard-1", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + { + Name: "dashboard-2", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{ + createFolderObject("team-a", ""), + } + + dashboard1 := createDashboardObjectWithFolder("dashboard-1", "team-a") + dashboard2 := createDashboardObject("dashboard-2") + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-1" && result.Action == repository.FileActionCreated + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-2" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil).Times(2) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + // Setup folder client + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + // Setup dashboard clients - need separate clients for Get calls + dashboard1Client := &mockDynamicInterface{ + items: []unstructured.Unstructured{dashboard1}, + } + dashboard2Client := &mockDynamicInterface{ + items: []unstructured.Unstructured{dashboard2}, + } + + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + // First call returns dashboard1Client, second returns dashboard2Client + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboard1Client, resources.DashboardResource, nil).Once() + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboard2Client, resources.DashboardResource, nil).Once() + + // Mock WriteResourceFileFromObject calls + // dashboard-1 is in team-a folder + repoResources.On("WriteResourceFileFromObject", mock.Anything, + mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-1" + }), + mock.MatchedBy(func(opts resources.WriteOptions) bool { + return opts.Path == "grafana/team-a" && opts.Ref == "feature/branch" + })).Return("grafana/team-a/dashboard-1.json", nil) + + // dashboard-2 has no folder + repoResources.On("WriteResourceFileFromObject", mock.Anything, + mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-2" + }), + mock.MatchedBy(func(opts resources.WriteOptions) bool { + return opts.Path == "grafana" && opts.Ref == "feature/branch" + })).Return("grafana/dashboard-2.json", nil) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_EmptyResources(t *testing.T) { + options := provisioningV0.ExportJobOptions{ + Path: "grafana", + Branch: "feature/branch", + Resources: []provisioningV0.ResourceRef{}, + } + + resourceClients := resources.NewMockResourceClients(t) + repoResources := resources.NewMockRepositoryResources(t) + mockProgress := jobs.NewMockJobProgressRecorder(t) + + err := ExportSpecificResources(context.Background(), options, resourceClients, repoResources, mockProgress) + require.EqualError(t, err, "no resources specified for export") +} + +func TestExportSpecificResources_RejectsFolders(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "my-folder", + Kind: "Folder", + Group: resources.FolderResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{} + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "my-folder" && + result.Action == repository.FileActionIgnored && + result.Error != nil && + result.Error.Error() == "folders are not supported for export" + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + // No ForKind or WriteResourceFileFromObject calls expected for folders + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_RejectsManagedResources(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "managed-dashboard", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{} + + dashboard := createDashboardObject("managed-dashboard") + meta, _ := utils.MetaAccessor(&dashboard) + meta.SetManagerProperties(utils.ManagerProperties{ + Kind: utils.ManagerKindRepo, + Identity: "some-repo", + }) + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "managed-dashboard" && + result.Action == repository.FileActionIgnored && + result.Error != nil && + result.Error.Error() == "resource dashboard.grafana.app/Dashboard/managed-dashboard is managed and cannot be exported" + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + dashboardClient := &mockDynamicInterface{ + items: []unstructured.Unstructured{dashboard}, + } + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboardClient, resources.DashboardResource, nil) + // No WriteResourceFileFromObject call expected for managed resources + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_RejectsUnsupportedResources(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "some-resource", + Kind: "Playlist", + Group: "playlist.grafana.app", + }, + } + + folderItems := []unstructured.Unstructured{} + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "some-resource" && + result.Action == repository.FileActionIgnored && + result.Error != nil && + result.Error.Error() == "resource type playlist.grafana.app/playlists is not supported for export" + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + // Mock ForKind to return a client (even though it's unsupported) + unsupportedClient := &mockDynamicInterface{items: []unstructured.Unstructured{}} + gvk := schema.GroupVersionKind{ + Group: "playlist.grafana.app", + Kind: "Playlist", + } + gvr := schema.GroupVersionResource{ + Group: "playlist.grafana.app", + Resource: "playlists", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(unsupportedClient, gvr, nil) + // No WriteResourceFileFromObject call expected for unsupported resources + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_FolderPathResolution(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "dashboard-in-nested-folder", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + // Create folder hierarchy: team-a -> subteam + folderItems := []unstructured.Unstructured{ + createFolderObject("team-a", ""), + createFolderObject("subteam", "team-a"), + } + + dashboard := createDashboardObjectWithFolder("dashboard-in-nested-folder", "subteam") + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-in-nested-folder" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + dashboardClient := &mockDynamicInterface{ + items: []unstructured.Unstructured{dashboard}, + } + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboardClient, resources.DashboardResource, nil) + + // Verify that the path includes the nested folder structure: grafana/team-a/subteam + repoResources.On("WriteResourceFileFromObject", mock.Anything, + mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-in-nested-folder" + }), + mock.MatchedBy(func(opts resources.WriteOptions) bool { + return opts.Path == "grafana/team-a/subteam" && opts.Ref == "feature/branch" + })).Return("grafana/team-a/subteam/dashboard-in-nested-folder.json", nil) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_FolderClientError(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "dashboard-1", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + resourceClients.On("Folder", mock.Anything).Return(nil, fmt.Errorf("folder client error")) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, nil, setupProgress, setupResources) + require.EqualError(t, err, "get folder client: folder client error") +} + +func TestExportSpecificResources_ResourceNotFound(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "non-existent-dashboard", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{} + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "non-existent-dashboard" && + result.Error != nil && + result.Error.Error() == "get resource dashboard.grafana.app/Dashboard/non-existent-dashboard: no items found" + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + // Empty client - resource not found + dashboardClient := &mockDynamicInterface{ + items: []unstructured.Unstructured{}, + } + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboardClient, resources.DashboardResource, nil) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_DashboardVersionConversion(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "v2-dashboard", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{} + + // Dashboard with storedVersion v2alpha1 + v1Dashboard := unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "v2-dashboard", + }, + "status": map[string]interface{}{ + "conversion": map[string]interface{}{ + "failed": true, + "storedVersion": "v2alpha1", + }, + }, + }, + } + + v2Dashboard := createV2DashboardObject("v2-dashboard", "v2alpha1") + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "v2-dashboard" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + // v1 client returns dashboard with storedVersion + v1Client := &mockDynamicInterface{ + items: []unstructured.Unstructured{v1Dashboard}, + } + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(v1Client, resources.DashboardResource, nil) + + // v2alpha1 client for fetching original version + v2Client := &mockDynamicInterface{ + items: []unstructured.Unstructured{v2Dashboard}, + } + v2GVR := schema.GroupVersionResource{ + Group: resources.DashboardResource.Group, + Version: "v2alpha1", + Resource: resources.DashboardResource.Resource, + } + resourceClients.On("ForResource", mock.Anything, v2GVR).Return(v2Client, gvk, nil) + + // Verify WriteResourceFileFromObject is called with v2 dashboard + repoResources.On("WriteResourceFileFromObject", mock.Anything, + mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "v2-dashboard" && + obj.GetAPIVersion() == "dashboard.grafana.app/v2alpha1" + }), + mock.MatchedBy(func(opts resources.WriteOptions) bool { + return opts.Path == "grafana" && opts.Ref == "feature/branch" + })).Return("grafana/v2-dashboard.json", nil) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.NoError(t, err) +} + +func TestExportSpecificResources_TooManyErrors(t *testing.T) { + resourceRefs := []provisioningV0.ResourceRef{ + { + Name: "dashboard-1", + Kind: "Dashboard", + Group: resources.DashboardResource.Group, + }, + } + + folderItems := []unstructured.Unstructured{} + + dashboard := createDashboardObject("dashboard-1") + + setupProgress := func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "exporting specific resources").Return() + progress.On("SetMessage", mock.Anything, "loading folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, mock.AnythingOfType("string")).Return().Maybe() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-1" && + result.Action == repository.FileActionIgnored && + result.Error != nil + })).Return() + progress.On("TooManyErrors").Return(fmt.Errorf("too many errors")) + } + + setupResources := func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients) { + folderClient := &mockDynamicInterface{items: folderItems} + resourceClients.On("Folder", mock.Anything).Return(folderClient, nil) + + dashboardClient := &mockDynamicInterface{ + items: []unstructured.Unstructured{dashboard}, + } + gvk := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Kind: "Dashboard", + } + resourceClients.On("ForKind", mock.Anything, gvk).Return(dashboardClient, resources.DashboardResource, nil) + + repoResources.On("WriteResourceFileFromObject", mock.Anything, + mock.Anything, + mock.Anything).Return("", fmt.Errorf("write error")) + } + + err := runExportSpecificResourcesTest(t, resourceRefs, folderItems, setupProgress, setupResources) + require.EqualError(t, err, "too many errors") +} diff --git a/pkg/registry/apis/provisioning/jobs/export/worker.go b/pkg/registry/apis/provisioning/jobs/export/worker.go index 092243c4567..3543b687c2d 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker.go @@ -100,6 +100,14 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, return fmt.Errorf("create repository resource client: %w", err) } + // Check if Resources list is provided (bulk export mode) + if len(options.Resources) > 0 { + progress.SetTotal(ctx, len(options.Resources)) + progress.StrictMaxErrors(1) // Fail fast on any error during export + return ExportSpecificResources(ctx, *options, clients, repositoryResources, progress) + } + + // Fall back to existing ExportAll behavior for backward compatibility return r.exportFn(ctx, cfg.Name, *options, clients, repositoryResources, progress) } diff --git a/pkg/registry/apis/provisioning/resources/resources.go b/pkg/registry/apis/provisioning/resources/resources.go index 4954e6b51b6..42613c598ab 100644 --- a/pkg/registry/apis/provisioning/resources/resources.go +++ b/pkg/registry/apis/provisioning/resources/resources.go @@ -167,6 +167,9 @@ func (r *ResourcesManager) WriteResourceFileFromObject(ctx context.Context, obj title = name } + fileName := slugify.Slugify(title) + ".json" + + // Use folder structure: get the folder path from the resource and append it to Path folder := meta.GetFolder() // Get the absolute path of the folder rootFolder := RootFolder(r.repo.Config()) @@ -184,13 +187,18 @@ func (r *ResourcesManager) WriteResourceFileFromObject(ctx context.Context, obj } } - fileName := slugify.Slugify(title) + ".json" + // Build the full path: start with options.Path, then add folder path, then filename + basePath := options.Path if fid.Path != "" { - fileName = safepath.Join(fid.Path, fileName) + if basePath != "" { + basePath = safepath.Join(basePath, fid.Path) + } else { + basePath = fid.Path + } } - if options.Path != "" { - fileName = safepath.Join(options.Path, fileName) + if basePath != "" { + fileName = safepath.Join(basePath, fileName) } parsed := ParsedResource{