Provisioning: bulk move by name (#108869)

* Add ResourceRef to move job spec
* Implement move by name
* Use map to struct
This commit is contained in:
Roberto Jiménez Sánchez
2025-07-30 10:30:24 +02:00
committed by GitHub
parent f4335fc5ce
commit f4e45bf3bc
11 changed files with 747 additions and 22 deletions
+6
View File
@@ -178,6 +178,12 @@ type MoveJobOptions struct {
// Destination path for the move (e.g. "new-location/")
TargetPath string `json:"targetPath,omitempty"`
// Resources to move
// This option has been created because currently the frontend does not use
// standarized app platform APIs. For performance and API consistency reasons, the preferred option
// is it to use the paths.
Resources []ResourceRef `json:"resources,omitempty"`
}
// The job status
@@ -512,6 +512,11 @@ func (in *MoveJobOptions) DeepCopyInto(out *MoveJobOptions) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make([]ResourceRef, len(*in))
copy(*out, *in)
}
return
}
@@ -1150,9 +1150,25 @@ func schema_pkg_apis_provisioning_v0alpha1_MoveJobOptions(ref common.ReferenceCa
Format: "",
},
},
"resources": {
SchemaProps: spec.SchemaProps{
Description: "Resources to move This option has been created because currently the frontend does not use standarized app platform APIs. For performance and API consistency reasons, the preferred option is it to use the paths.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceRef"),
},
},
},
},
},
},
},
},
Dependencies: []string{
"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.ResourceRef"},
}
}
@@ -7,6 +7,7 @@ API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provis
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,JobStatus,Summary
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ManagerStats,Stats
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,MoveJobOptions,Paths
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,MoveJobOptions,Resources
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RefList,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositoryList,Items
API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,Workflows
@@ -167,12 +167,12 @@ func deduplicatePaths(paths []string) []string {
return paths
}
seen := make(map[string]bool, len(paths))
seen := make(map[string]struct{}, len(paths))
result := make([]string, 0, len(paths))
for _, path := range paths {
if !seen[path] {
seen[path] = true
if _, exists := seen[path]; !exists {
seen[path] = struct{}{}
result = append(result, path)
}
}
@@ -7,21 +7,26 @@ import (
"path/filepath"
"time"
"k8s.io/apimachinery/pkg/runtime/schema"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
)
type Worker struct {
syncWorker jobs.Worker
wrapFn repository.WrapWithStageFn
syncWorker jobs.Worker
wrapFn repository.WrapWithStageFn
resourcesFactory resources.RepositoryResourcesFactory
}
func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn) *Worker {
func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory) *Worker {
return &Worker{
syncWorker: syncWorker,
wrapFn: wrapFn,
syncWorker: syncWorker,
wrapFn: wrapFn,
resourcesFactory: resourcesFactory,
}
}
@@ -45,7 +50,8 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
}
paths := opts.Paths
progress.SetTotal(ctx, len(paths))
progress.SetTotal(ctx, len(paths)+len(opts.Resources))
progress.StrictMaxErrors(1) // Fail fast on any error during move
fn := func(repo repository.Repository, _ bool) error {
@@ -54,6 +60,18 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
return errors.New("move job submitted targeting repository that is not a ReaderWriter")
}
// Resolve ResourceRef entries to file paths using RepositoryResources
if len(opts.Resources) > 0 {
resolvedPaths, err := w.resolveResourcesToPaths(ctx, rw, progress, opts.Resources)
if err != nil {
return err
}
paths = append(paths, resolvedPaths...)
}
// Deduplicate paths to avoid attempting to move the same file multiple times
paths = deduplicatePaths(paths)
return w.moveFiles(ctx, rw, progress, opts, paths...)
}
@@ -125,3 +143,67 @@ func (w *Worker) constructTargetPath(jobTargetPath, sourcePath string) string {
// For files, just append the filename
return jobTargetPath + fileName
}
// resolveResourcesToPaths converts ResourceRef entries to file paths, recording errors for individual resources
func (w *Worker) resolveResourcesToPaths(ctx context.Context, rw repository.ReaderWriter, progress jobs.JobProgressRecorder, resources []provisioning.ResourceRef) ([]string, error) {
if len(resources) == 0 {
return nil, nil
}
progress.SetMessage(ctx, "Resolving resource paths")
repositoryResources, err := w.resourcesFactory.Client(ctx, rw)
if err != nil {
return nil, fmt.Errorf("create repository resources client: %w", err)
}
resolvedPaths := make([]string, 0, len(resources))
for _, resource := range resources {
result := jobs.JobResourceResult{
Name: resource.Name,
Group: resource.Group,
Action: repository.FileActionRenamed, // Will be used for move later
}
gvk := schema.GroupVersionKind{
Group: resource.Group,
Kind: resource.Kind,
// Version is left empty so ForKind will use the preferred version
}
progress.SetMessage(ctx, fmt.Sprintf("Finding path for resource %s/%s/%s", resource.Group, resource.Kind, resource.Name))
resourcePath, err := repositoryResources.FindResourcePath(ctx, resource.Name, gvk)
if err != nil {
result.Error = fmt.Errorf("find path for resource %s/%s/%s: %w", resource.Group, resource.Kind, resource.Name, err)
progress.Record(ctx, result)
// Continue with next resource instead of failing fast
if err := progress.TooManyErrors(); err != nil {
return resolvedPaths, err
}
continue
}
result.Path = resourcePath
resolvedPaths = append(resolvedPaths, resourcePath)
}
return resolvedPaths, nil
}
// deduplicatePaths removes duplicate file paths from the slice while preserving order
func deduplicatePaths(paths []string) []string {
if len(paths) <= 1 {
return paths
}
seen := make(map[string]struct{}, len(paths))
result := make([]string, 0, len(paths))
for _, path := range paths {
if _, exists := seen[path]; !exists {
seen[path] = struct{}{}
result = append(result, path)
}
}
return result
}
@@ -3,15 +3,18 @@ package move
import (
"context"
"errors"
"fmt"
"path/filepath"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@@ -72,7 +75,7 @@ func TestMoveWorker_IsSupported(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
result := worker.IsSupported(context.Background(), tt.job)
require.Equal(t, tt.expected, result)
})
@@ -86,7 +89,7 @@ func TestMoveWorker_ProcessMissingMoveSettings(t *testing.T) {
},
}
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
err := worker.Process(context.Background(), nil, job, nil)
require.EqualError(t, err, "missing move settings")
}
@@ -101,7 +104,7 @@ func TestMoveWorker_ProcessMissingTargetPath(t *testing.T) {
},
}
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
err := worker.Process(context.Background(), nil, job, nil)
require.EqualError(t, err, "target path is required for move operation")
}
@@ -117,7 +120,7 @@ func TestMoveWorker_ProcessInvalidTargetPath(t *testing.T) {
},
}
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
err := worker.Process(context.Background(), nil, job, nil)
require.EqualError(t, err, "target path must be a directory (should end with '/')")
}
@@ -149,7 +152,7 @@ func TestMoveWorker_ProcessNotReaderWriter(t *testing.T) {
mockProgress.On("SetTotal", mock.Anything, 1).Return()
mockProgress.On("StrictMaxErrors", 1).Return()
worker := NewWorker(nil, mockWrapFn.Execute)
worker := NewWorker(nil, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.EqualError(t, err, "move files in repository: move job submitted targeting repository that is not a ReaderWriter")
}
@@ -173,7 +176,7 @@ func TestMoveWorker_ProcessWrapFnError(t *testing.T) {
mockProgress.On("SetTotal", mock.Anything, 1).Return()
mockProgress.On("StrictMaxErrors", 1).Return()
worker := NewWorker(nil, mockWrapFn.Execute)
worker := NewWorker(nil, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.EqualError(t, err, "move files in repository: stage failed")
}
@@ -220,7 +223,7 @@ func TestMoveWorker_ProcessMoveFilesSuccess(t *testing.T) {
return result.Path == "test/path2" && result.Action == repository.FileActionRenamed && result.Error == nil
})).Return()
worker := NewWorker(nil, mockWrapFn.Execute)
worker := NewWorker(nil, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.NoError(t, err)
}
@@ -259,7 +262,7 @@ func TestMoveWorker_ProcessMoveFilesWithError(t *testing.T) {
})).Return()
mockProgress.On("TooManyErrors").Return(errors.New("too many errors"))
worker := NewWorker(nil, mockWrapFn.Execute)
worker := NewWorker(nil, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.EqualError(t, err, "move files in repository: too many errors")
}
@@ -304,7 +307,7 @@ func TestMoveWorker_ProcessWithSyncWorker(t *testing.T) {
return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental
}), mockProgress).Return(nil)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.NoError(t, err)
}
@@ -345,7 +348,7 @@ func TestMoveWorker_ProcessSyncWorkerError(t *testing.T) {
syncError := errors.New("sync failed")
mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.Anything, mockProgress).Return(syncError)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.EqualError(t, err, "pull resources: sync failed")
}
@@ -426,7 +429,7 @@ func TestMoveWorker_moveFiles(t *testing.T) {
}
}
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
err := worker.moveFiles(context.Background(), mockRepo, mockProgress, opts, tt.paths...)
if tt.expectedError != "" {
@@ -482,9 +485,351 @@ func TestMoveWorker_constructTargetPath(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
worker := NewWorker(nil, nil)
worker := NewWorker(nil, nil, nil)
result := worker.constructTargetPath(tt.jobTargetPath, tt.sourcePath)
require.Equal(t, tt.expectedTarget, result)
})
}
}
func TestMoveWorker_ProcessWithResourceReferences(t *testing.T) {
job := provisioning.Job{
Spec: provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
Paths: []string{"test/path1"},
TargetPath: "new/location/",
Resources: []provisioning.ResourceRef{
{
Name: "dashboard-uid",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
},
}
mockRepo := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
}
mockProgress := jobs.NewMockJobProgressRecorder(t)
mockWrapFn := repository.NewMockWrapWithStageFn(t)
mockResourcesFactory := resources.NewMockRepositoryResourcesFactory(t)
mockRepoResources := resources.NewMockRepositoryResources(t)
mockSyncWorker := jobs.NewMockWorker(t)
mockWrapFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOptions repository.StageOptions, fn func(repository.Repository, bool) error) error {
return fn(mockRepo, false)
})
mockProgress.On("SetTotal", mock.Anything, 2).Return() // 1 path + 1 resource
mockProgress.On("StrictMaxErrors", 1).Return()
mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return()
mockProgress.On("SetMessage", mock.Anything, "Finding path for resource dashboard.grafana.app/Dashboard/dashboard-uid").Return()
mockProgress.On("SetMessage", mock.Anything, "Moving test/path1 to new/location/path1").Return()
mockProgress.On("SetMessage", mock.Anything, "Moving dashboard/file.yaml to new/location/file.yaml").Return()
mockProgress.On("TooManyErrors").Return(nil).Times(2)
mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(mockRepoResources, nil)
mockRepoResources.On("FindResourcePath", mock.Anything, "dashboard-uid", schema.GroupVersionKind{
Group: "dashboard.grafana.app",
Kind: "Dashboard",
}).Return("dashboard/file.yaml", nil)
mockRepo.On("Move", mock.Anything, "test/path1", "new/location/path1", "", "Move test/path1 to new/location/path1").Return(nil)
mockRepo.On("Move", mock.Anything, "dashboard/file.yaml", "new/location/file.yaml", "", "Move dashboard/file.yaml to new/location/file.yaml").Return(nil)
mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Path == "test/path1" && result.Action == repository.FileActionRenamed && result.Error == nil
})).Return()
mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Path == "dashboard/file.yaml" && result.Action == repository.FileActionRenamed && result.Error == nil
})).Return()
// Add expectations for sync worker (called when ref is empty)
mockProgress.On("ResetResults").Return()
mockProgress.On("SetMessage", mock.Anything, "pull resources").Return()
mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.MatchedBy(func(syncJob provisioning.Job) bool {
return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental
}), mockProgress).Return(nil)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.NoError(t, err)
}
func TestMoveWorker_ProcessResourceReferencesError(t *testing.T) {
job := provisioning.Job{
Spec: provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "new/location/",
Resources: []provisioning.ResourceRef{
{
Name: "non-existent-uid",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
},
}
mockRepo := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
}
mockProgress := jobs.NewMockJobProgressRecorder(t)
mockWrapFn := repository.NewMockWrapWithStageFn(t)
mockResourcesFactory := resources.NewMockRepositoryResourcesFactory(t)
mockRepoResources := resources.NewMockRepositoryResources(t)
mockWrapFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOptions repository.StageOptions, fn func(repository.Repository, bool) error) error {
return fn(mockRepo, false)
})
mockProgress.On("SetTotal", mock.Anything, 1).Return() // 1 resource
mockProgress.On("StrictMaxErrors", 1).Return()
mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return()
mockProgress.On("SetMessage", mock.Anything, "Finding path for resource dashboard.grafana.app/Dashboard/non-existent-uid").Return()
mockProgress.On("TooManyErrors").Return(nil)
mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(mockRepoResources, nil)
resourceError := errors.New("resource not found")
mockRepoResources.On("FindResourcePath", mock.Anything, "non-existent-uid", schema.GroupVersionKind{
Group: "dashboard.grafana.app",
Kind: "Dashboard",
}).Return("", resourceError)
mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Name == "non-existent-uid" && result.Group == "dashboard.grafana.app" &&
result.Action == repository.FileActionRenamed &&
result.Error != nil && result.Error.Error() == "find path for resource dashboard.grafana.app/Dashboard/non-existent-uid: resource not found"
})).Return()
// Add expectations for sync worker (called when ref is empty)
mockSyncWorker := jobs.NewMockWorker(t)
mockProgress.On("ResetResults").Return()
mockProgress.On("SetMessage", mock.Anything, "pull resources").Return()
mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.MatchedBy(func(syncJob provisioning.Job) bool {
return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental
}), mockProgress).Return(nil)
worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.NoError(t, err) // Should continue despite individual resource errors
}
func TestMoveWorker_ProcessResourcesFactoryError(t *testing.T) {
job := provisioning.Job{
Spec: provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "new/location/",
Resources: []provisioning.ResourceRef{
{
Name: "dashboard-uid",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
},
}
mockRepo := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
}
mockProgress := jobs.NewMockJobProgressRecorder(t)
mockWrapFn := repository.NewMockWrapWithStageFn(t)
mockResourcesFactory := resources.NewMockRepositoryResourcesFactory(t)
mockWrapFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOptions repository.StageOptions, fn func(repository.Repository, bool) error) error {
return fn(mockRepo, false)
})
mockProgress.On("SetTotal", mock.Anything, 1).Return() // 1 resource
mockProgress.On("StrictMaxErrors", 1).Return()
mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return()
factoryError := errors.New("failed to create resources client")
mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(nil, factoryError)
worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory)
err := worker.Process(context.Background(), mockRepo, job, mockProgress)
require.EqualError(t, err, "move files in repository: create repository resources client: failed to create resources client")
}
func TestMoveWorker_resolveResourcesToPaths(t *testing.T) {
tests := []struct {
name string
resources []provisioning.ResourceRef
resourcePaths map[string]string
resourceErrors map[string]error
tooManyErrors error
expectedPaths []string
expectedError string
expectContinue bool
}{
{
name: "single resource success",
resources: []provisioning.ResourceRef{
{
Name: "dashboard-uid",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
resourcePaths: map[string]string{
"dashboard-uid": "dashboards/test.json",
},
expectedPaths: []string{"dashboards/test.json"},
},
{
name: "multiple resources success",
resources: []provisioning.ResourceRef{
{
Name: "dashboard1",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
{
Name: "dashboard2",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
resourcePaths: map[string]string{
"dashboard1": "dashboards/dash1.json",
"dashboard2": "dashboards/dash2.json",
},
expectedPaths: []string{"dashboards/dash1.json", "dashboards/dash2.json"},
},
{
name: "resource not found continues",
resources: []provisioning.ResourceRef{
{
Name: "non-existent",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
{
Name: "existing",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
resourcePaths: map[string]string{
"existing": "dashboards/existing.json",
},
resourceErrors: map[string]error{
"non-existent": errors.New("not found"),
},
expectedPaths: []string{"dashboards/existing.json"},
expectContinue: true,
},
{
name: "empty resources list",
resources: []provisioning.ResourceRef{},
expectedPaths: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockRepo := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
}
mockProgress := jobs.NewMockJobProgressRecorder(t)
mockResourcesFactory := resources.NewMockRepositoryResourcesFactory(t)
mockRepoResources := resources.NewMockRepositoryResources(t)
if len(tt.resources) > 0 {
mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return()
mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(mockRepoResources, nil)
for _, resource := range tt.resources {
mockProgress.On("SetMessage", mock.Anything, fmt.Sprintf("Finding path for resource %s/%s/%s", resource.Group, resource.Kind, resource.Name)).Return()
gvk := schema.GroupVersionKind{
Group: resource.Group,
Kind: resource.Kind,
}
if path, ok := tt.resourcePaths[resource.Name]; ok {
mockRepoResources.On("FindResourcePath", mock.Anything, resource.Name, gvk).Return(path, nil)
} else if err, ok := tt.resourceErrors[resource.Name]; ok {
mockRepoResources.On("FindResourcePath", mock.Anything, resource.Name, gvk).Return("", err)
mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Name == resource.Name && result.Group == resource.Group &&
result.Action == repository.FileActionRenamed && result.Error != nil
})).Return()
if tt.tooManyErrors != nil {
mockProgress.On("TooManyErrors").Return(tt.tooManyErrors)
} else {
mockProgress.On("TooManyErrors").Return(nil)
}
}
}
}
worker := NewWorker(nil, nil, mockResourcesFactory)
paths, err := worker.resolveResourcesToPaths(context.Background(), mockRepo, mockProgress, tt.resources)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
} else {
require.NoError(t, err)
}
require.Equal(t, tt.expectedPaths, paths)
mockResourcesFactory.AssertExpectations(t)
if len(tt.resources) > 0 {
mockRepoResources.AssertExpectations(t)
}
mockProgress.AssertExpectations(t)
})
}
}
func TestMoveWorker_deduplicatePaths(t *testing.T) {
tests := []struct {
name string
input []string
expected []string
}{
{
name: "no duplicates",
input: []string{"path1", "path2", "path3"},
expected: []string{"path1", "path2", "path3"},
},
{
name: "with duplicates",
input: []string{"path1", "path2", "path1", "path3", "path2"},
expected: []string{"path1", "path2", "path3"},
},
{
name: "empty slice",
input: []string{},
expected: []string{},
},
{
name: "single item",
input: []string{"path1"},
expected: []string{"path1"},
},
{
name: "all duplicates",
input: []string{"path1", "path1", "path1"},
expected: []string{"path1"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := deduplicatePaths(tt.input)
require.Equal(t, tt.expected, result)
})
}
}
+1 -1
View File
@@ -619,7 +619,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
)
deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources)
moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible)
moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources)
workers := []jobs.Worker{
deleteWorker,
exportWorker,
@@ -3253,6 +3253,18 @@
"description": "Ref to the branch or commit hash that should move",
"type": "string"
},
"resources": {
"description": "Resources to move This option has been created because currently the frontend does not use standarized app platform APIs. For performance and API consistency reasons, the preferred option is it to use the paths.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ResourceRef"
}
]
}
},
"targetPath": {
"description": "Destination path for the move (e.g. \"new-location/\")",
"type": "string"
@@ -1568,6 +1568,200 @@ func TestIntegrationProvisioning_MoveJob(t *testing.T) {
assert.Equal(collect, "error", state, "move job should have failed due to missing target path")
}, time.Second*10, time.Millisecond*100, "Expected move job to fail with error state")
})
t.Run("move by resource reference", func(t *testing.T) {
// Create a unique repository for resource reference testing to avoid contamination
const refRepo = "move-ref-test-repo"
localRefTmp := helper.RenderObject(t, "testdata/local-write.json.tmpl", map[string]any{
"Name": refRepo,
"SyncEnabled": true,
"SyncTarget": "folder",
})
_, err := helper.Repositories.Resource.Create(ctx, localRefTmp, metav1.CreateOptions{})
require.NoError(t, err)
// Create modified test files with unique UIDs for ResourceRef testing
allPanelsContent := helper.LoadFile("testdata/all-panels.json")
textOptionsContent := helper.LoadFile("testdata/text-options.json")
timelineDemoContent := helper.LoadFile("testdata/timeline-demo.json")
// Modify UIDs to be unique for ResourceRef tests
allPanelsModified := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "moveref1"`, 1)
textOptionsModified := strings.Replace(string(textOptionsContent), `"uid": "WZ7AhQiVz"`, `"uid": "moveref2"`, 1)
timelineDemoModified := strings.Replace(string(timelineDemoContent), `"uid": "mIJjFy8Kz"`, `"uid": "moveref3"`, 1)
// Create temporary files and copy them to the provisioning path
tmpDir := t.TempDir()
tmpFile1 := filepath.Join(tmpDir, "move-ref-test-1.json")
tmpFile2 := filepath.Join(tmpDir, "move-ref-test-2.json")
tmpFile3 := filepath.Join(tmpDir, "move-ref-test-3.json")
require.NoError(t, os.WriteFile(tmpFile1, []byte(allPanelsModified), 0644))
require.NoError(t, os.WriteFile(tmpFile2, []byte(textOptionsModified), 0644))
require.NoError(t, os.WriteFile(tmpFile3, []byte(timelineDemoModified), 0644))
// Copy files to provisioning path to set up test - use refRepo's path
helper.CopyToProvisioningPath(t, tmpFile1, "move-source-1.json")
helper.CopyToProvisioningPath(t, tmpFile2, "move-source-2.json")
helper.CopyToProvisioningPath(t, tmpFile3, "move-source-3.json")
// Sync to populate resources in Grafana
helper.SyncAndWait(t, refRepo, nil)
t.Run("move single dashboard by resource reference", func(t *testing.T) {
// Create move job for single dashboard using ResourceRef
result := helper.AdminREST.Post().
Namespace("default").
Resource("repositories").
Name(refRepo).
SubResource("jobs").
Body(asJSON(&provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "moved-by-ref/",
Resources: []provisioning.ResourceRef{
{
Name: "moveref1", // UID from modified all-panels.json
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
})).
SetHeader("Content-Type", "application/json").
Do(ctx)
require.NoError(t, result.Error(), "should be able to create move job with ResourceRef")
// Wait for job to complete
helper.AwaitJobs(t, refRepo)
// Verify corresponding file is moved in repository
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "moved-by-ref", "move-source-1.json")
require.NoError(t, err, "file should be moved to new location in repository")
// Verify original file is deleted from repository
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "move-source-1.json")
require.Error(t, err, "original file should be deleted from repository")
require.True(t, apierrors.IsNotFound(err), "should be not found error")
// Verify dashboard still exists in Grafana (should be updated after sync)
helper.SyncAndWait(t, refRepo, nil)
_, err = helper.DashboardsV1.Resource.Get(ctx, "moveref1", metav1.GetOptions{})
require.NoError(t, err, "dashboard should still exist in Grafana after move")
// Verify other resource still exists in original location
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "move-source-2.json")
require.NoError(t, err, "other files should still exist in original location")
})
t.Run("move multiple resources by reference", func(t *testing.T) {
// Create move job for remaining resource using ResourceRef
result := helper.AdminREST.Post().
Namespace("default").
Resource("repositories").
Name(refRepo).
SubResource("jobs").
Body(asJSON(&provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "archived-by-ref/",
Resources: []provisioning.ResourceRef{
{
Name: "moveref2", // UID from modified text-options.json
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
})).
SetHeader("Content-Type", "application/json").
Do(ctx)
require.NoError(t, result.Error(), "should be able to create move job with ResourceRef")
// Wait for job to complete
helper.AwaitJobs(t, refRepo)
// Verify file is moved in repository
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "archived-by-ref", "move-source-2.json")
require.NoError(t, err, "file should be moved to new location")
// Verify original file is deleted from repository
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "move-source-2.json")
require.Error(t, err, "original file should be deleted from repository")
require.True(t, apierrors.IsNotFound(err), "should be not found error")
// Verify dashboard still exists in Grafana after sync
helper.SyncAndWait(t, refRepo, nil)
_, err = helper.DashboardsV1.Resource.Get(ctx, "moveref2", metav1.GetOptions{})
require.NoError(t, err, "dashboard should still exist in Grafana after move")
})
t.Run("mixed move - paths and resources", func(t *testing.T) {
// Setup fresh resources for mixed test
tmpMixed1 := filepath.Join(tmpDir, "mixed-move-1.json")
tmpMixed2 := filepath.Join(tmpDir, "mixed-move-2.json")
allPanelsMixed := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "mixedmove1"`, 1)
textOptionsMixed := strings.Replace(string(textOptionsContent), `"uid": "WZ7AhQiVz"`, `"uid": "mixedmove2"`, 1)
require.NoError(t, os.WriteFile(tmpMixed1, []byte(allPanelsMixed), 0644))
require.NoError(t, os.WriteFile(tmpMixed2, []byte(textOptionsMixed), 0644))
helper.CopyToProvisioningPath(t, tmpMixed1, "mixed-move-1.json") // UID: mixedmove1
helper.CopyToProvisioningPath(t, tmpMixed2, "mixed-move-2.json") // UID: mixedmove2
helper.SyncAndWait(t, refRepo, nil)
// Create move job that combines both paths and resource references
result := helper.AdminREST.Post().
Namespace("default").
Resource("repositories").
Name(refRepo).
SubResource("jobs").
Body(asJSON(&provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "mixed-target/",
Paths: []string{"mixed-move-1.json"}, // Move by path
Resources: []provisioning.ResourceRef{
{
Name: "mixedmove2", // Move by resource reference
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
})).
SetHeader("Content-Type", "application/json").
Do(ctx)
require.NoError(t, result.Error(), "should be able to create mixed move job")
// Wait for job to complete
helper.AwaitJobs(t, refRepo)
// Verify both targeted resources are moved in repository
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "mixed-target", "mixed-move-1.json")
require.NoError(t, err, "file moved by path should exist at new location")
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "mixed-target", "mixed-move-2.json")
require.NoError(t, err, "file moved by resource ref should exist at new location")
// Verify files are deleted from original locations
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "mixed-move-1.json")
require.Error(t, err, "file moved by path should be deleted from original location")
_, err = helper.Repositories.Resource.Get(ctx, refRepo, metav1.GetOptions{}, "files", "mixed-move-2.json")
require.Error(t, err, "file moved by resource ref should be deleted from original location")
// Verify dashboards still exist in Grafana after sync
helper.SyncAndWait(t, refRepo, nil)
_, err = helper.DashboardsV1.Resource.Get(ctx, "mixedmove1", metav1.GetOptions{})
require.NoError(t, err, "dashboard moved by path should still exist in Grafana")
_, err = helper.DashboardsV1.Resource.Get(ctx, "mixedmove2", metav1.GetOptions{})
require.NoError(t, err, "dashboard moved by resource ref should still exist in Grafana")
})
})
}
func TestIntegrationProvisioning_MoveResources(t *testing.T) {
@@ -1850,4 +2044,66 @@ func TestIntegrationProvisioning_MoveResources(t *testing.T) {
require.Error(t, result.Error(), "should fail when source file doesn't exist")
})
})
t.Run("move non-existent resource by reference", func(t *testing.T) {
// Create move job for non-existent resource
result := helper.AdminREST.Post().
Namespace("default").
Resource("repositories").
Name(repo).
SubResource("jobs").
Body(asJSON(&provisioning.JobSpec{
Action: provisioning.JobActionMove,
Move: &provisioning.MoveJobOptions{
TargetPath: "moved-nonexistent/",
Resources: []provisioning.ResourceRef{
{
Name: "non-existent-move-uid",
Kind: "Dashboard",
Group: "dashboard.grafana.app",
},
},
},
})).
SetHeader("Content-Type", "application/json").
Do(ctx)
require.NoError(t, result.Error(), "should be able to create move job")
// Wait for job to complete - should record error but continue
require.EventuallyWithT(t, func(collect *assert.CollectT) {
list := &unstructured.UnstructuredList{}
err := helper.AdminREST.Get().
Namespace("default").
Resource("repositories").
Name(repo).
SubResource("jobs").
Do(ctx).Into(list)
assert.NoError(collect, err, "should be able to list jobs")
assert.NotEmpty(collect, list.Items, "expect at least one job")
// Find the most recent move job
var moveJob *unstructured.Unstructured
for _, elem := range list.Items {
assert.Equal(collect, repo, elem.GetLabels()["provisioning.grafana.app/repository"], "should have repo label")
action := mustNestedString(elem.Object, "spec", "action")
if action == "move" {
// Get the most recent one (they should be ordered by creation time)
moveJob = &elem
}
}
if !assert.NotNil(collect, moveJob, "should find a move job") {
return
}
state := mustNestedString(moveJob.Object, "status", "state")
// The job should complete but record errors for individual resource resolution failures
if state == "error" || state == "completed" || state == "success" {
// Any of these states is acceptable - the key is that resource resolution errors are recorded
// and don't fail the entire job due to error-tolerant implementation
return
}
assert.Fail(collect, "job should complete or error, but got state: %s", state)
}, time.Second*10, time.Millisecond*100, "Expected move job to handle non-existent resource")
})
}