From 837f4864b1c2966e03407379ce86928daaddae53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Jim=C3=A9nez=20S=C3=A1nchez?= Date: Wed, 9 Apr 2025 12:14:43 +0200 Subject: [PATCH] Provisioning: unit test export job (#103620) * Add repository resources interface for export worker * Add mocks for repository resources * Add unit tests for ExportWorker's IsSupported method * Add unit tests for ExportWorker's Process method, covering scenarios for missing export settings, write permissions, branch restrictions, and client creation failures. * Fix unit tests * Single function * Add more unit tests * Add test for failed folder * Fail export folder errors * Add another test * Positive folder export * Too many folder export errors * Too many errors on folder export * Partial folder errors * Add test for nested folder * Add test dashboard export * More cases * Ignore existing dashboards * Fix folder tests * Fix clonable test * Add clone failure test * Add test clean up without push * Working tests * Use mock clonable * Add unit tests for IsWriteAllowed * Add behaviour to cover ref equal to configured branch * Fix worker test * Fix linting * Split clone and push * Wrapper for clone and push * Separate methods for resources export * Separate folder export * Simplify single signature * Refactor a bit more * Separate folder export function * Split it into different files * Add FIXME * Export function mock * Export Resources tests * Add test for cannot find client * Check for branch * Fix registry * Move folder export tests * Pass wrapper function * Add worker tests * Fail if branch is passed for clonable repositories * Fix merge issues --- .../apis/provisioning/jobs/export/all.go | 22 + .../apis/provisioning/jobs/export/folders.go | 60 + .../provisioning/jobs/export/folders_test.go | 379 +++++ .../jobs/export/mock_export_fn.go | 95 ++ .../jobs/export/mock_wrap_with_clone_fn.go | 87 ++ .../provisioning/jobs/export/resources.go | 68 + .../jobs/export/resources_test.go | 311 ++++ .../apis/provisioning/jobs/export/worker.go | 114 +- .../provisioning/jobs/export/worker_test.go | 1253 +++-------------- pkg/registry/apis/provisioning/register.go | 3 + 10 files changed, 1205 insertions(+), 1187 deletions(-) create mode 100644 pkg/registry/apis/provisioning/jobs/export/all.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/folders.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/folders_test.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/mock_export_fn.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/mock_wrap_with_clone_fn.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/resources.go create mode 100644 pkg/registry/apis/provisioning/jobs/export/resources_test.go diff --git a/pkg/registry/apis/provisioning/jobs/export/all.go b/pkg/registry/apis/provisioning/jobs/export/all.go new file mode 100644 index 00000000000..3824dae0331 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/all.go @@ -0,0 +1,22 @@ +package export + +import ( + "context" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "k8s.io/client-go/dynamic" +) + +func ExportAll(ctx context.Context, repoName string, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, folderClient dynamic.ResourceInterface, progress jobs.JobProgressRecorder) error { + if err := ExportFolders(ctx, repoName, options, folderClient, repositoryResources, progress); err != nil { + return err + } + + if err := ExportResources(ctx, options, clients, repositoryResources, progress); err != nil { + return err + } + + return nil +} diff --git a/pkg/registry/apis/provisioning/jobs/export/folders.go b/pkg/registry/apis/provisioning/jobs/export/folders.go new file mode 100644 index 00000000000..b2cce228812 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/folders.go @@ -0,0 +1,60 @@ +package export + +import ( + "context" + "errors" + "fmt" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" +) + +func ExportFolders(ctx context.Context, repoName string, options provisioning.ExportJobOptions, folderClient dynamic.ResourceInterface, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + // Load and write all folders + // FIXME: we load the entire tree in memory + progress.SetMessage(ctx, "read folder tree from API server") + + tree := resources.NewEmptyFolderTree() + if err := resources.ForEach(ctx, folderClient, func(item *unstructured.Unstructured) error { + if tree.Count() >= resources.MaxNumberOfFolders { + return errors.New("too many folders") + } + + // FIXME: repoName should be part of skip folder export + return tree.AddUnstructured(item, repoName) + }); err != nil { + return fmt.Errorf("load folder tree: %w", err) + } + + progress.SetMessage(ctx, "write folders to repository") + err := repositoryResources.EnsureFolderTreeExists(ctx, options.Branch, options.Path, tree, func(folder resources.Folder, created bool, err error) error { + result := jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Name: folder.ID, + Resource: resources.FolderResource.Resource, + Group: resources.FolderResource.Group, + Path: folder.Path, + Error: err, + } + + if !created { + result.Action = repository.FileActionIgnored + } + + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + + return nil + }) + if err != nil { + return fmt.Errorf("write folders to repository: %w", err) + } + + return nil +} diff --git a/pkg/registry/apis/provisioning/jobs/export/folders_test.go b/pkg/registry/apis/provisioning/jobs/export/folders_test.go new file mode 100644 index 00000000000..fdcc54d282f --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/folders_test.go @@ -0,0 +1,379 @@ +package export + +import ( + "context" + "errors" + "fmt" + "testing" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + mock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8testing "k8s.io/client-go/testing" +) + +func TestExportFolders(t *testing.T) { + tests := []struct { + name string + reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) + expectedError string + setupProgress func(progress *jobs.MockJobProgressRecorder) + setupResources func(repoResources *resources.MockRepositoryResources) + }{ + { + name: "list folders error", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("failed to list folders") + }, + expectedError: "load folder tree: error executing list: failed to list folders", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, mock.Anything).Return() + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + }, + }, + { + name: "too many folders", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + list := &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "FolderList", + }, + Items: make([]metav1.PartialObjectMetadata, resources.MaxNumberOfFolders+1), + } + for i := 0; i <= resources.MaxNumberOfFolders; i++ { + list.Items[i] = metav1.PartialObjectMetadata{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("folder-%d", i), + }, + } + } + return true, list, nil + }, + expectedError: "load folder tree: too many folders", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, mock.Anything).Return() + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + }, + }, + { + name: "ensure folder tree error", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + // Return empty list to get past the folder loading + return true, &metav1.PartialObjectMetadataList{}, nil + }, + expectedError: "write folders to repository: failed to ensure folder tree", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, mock.Anything).Return() + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + repoResources.On("EnsureFolderTreeExists", mock.Anything, "feature/branch", "grafana", mock.Anything, mock.Anything).Return(fmt.Errorf("failed to ensure folder tree")) + }, + }, + { + name: "successful folder migration", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + list := &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "FolderList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "folder-1", + Annotations: map[string]string{ + "folder.grafana.app/uid": "folder-1-uid", + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "folder-2", + Annotations: map[string]string{ + "folder.grafana.app/uid": "folder-2-uid", + }, + }, + }, + }, + } + return true, list, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, "write folders to repository").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "folder-1-uid" && result.Action == repository.FileActionCreated + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "folder-2-uid" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + repoResources.On("EnsureFolderTreeExists", mock.Anything, "feature/branch", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { + return tree.Count() == 2 + }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { + require.NoError(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, true, nil)) + require.NoError(t, fn(resources.Folder{ID: "folder-2-uid", Path: "grafana/folder-2"}, true, nil)) + + return true + })).Return(nil) + }, + }, + { + name: "successful folder migration with resource export errors", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + list := &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "FolderList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "folder-1", + Annotations: map[string]string{ + "folder.grafana.app/uid": "folder-1-uid", + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "folder-2", + Annotations: map[string]string{ + "folder.grafana.app/uid": "folder-2-uid", + }, + }, + }, + }, + } + return true, list, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, "write folders to repository").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "folder-1-uid" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "didn't work" + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "folder-2-uid" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + repoResources.On("EnsureFolderTreeExists", mock.Anything, "feature/branch", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { + return tree.Count() == 2 + }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { + require.NoError(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, false, errors.New("didn't work"))) + require.NoError(t, fn(resources.Folder{ID: "folder-2-uid", Path: "grafana/folder-2"}, true, nil)) + + return true + })).Return(nil) + }, + }, + { + name: "too many errors", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + list := &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "FolderList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "folder-1", + Annotations: map[string]string{ + "folder.grafana.app/uid": "folder-1-uid", + }, + }, + }, + }, + } + return true, list, nil + }, + expectedError: "write folders to repository: too many errors encountered", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, "write folders to repository").Return() + progress.On("Record", mock.Anything, mock.Anything).Return() + progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered")) + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + repoResources.On("EnsureFolderTreeExists", mock.Anything, "feature/branch", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { + return tree.Count() == 1 + }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { + require.Error(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, true, nil), "too many errors encountered") + return true + })).Return(fmt.Errorf("too many errors encountered")) + }, + }, + { + name: "successful nested folder migration", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + if action.GetResource() == resources.DashboardResource { + // Return empty dashboard list + return true, &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "FolderList", + }, + }, nil + } + + list := &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "FolderList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent-folder", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.FolderResource.GroupVersion().String(), + Kind: "Folder", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "child-folder", + Annotations: map[string]string{ + "grafana.app/folder": "parent-folder", + }, + }, + }, + }, + } + return true, list, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() + progress.On("SetMessage", mock.Anything, "write folders to repository").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "parent-uid" && result.Action == repository.FileActionCreated + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "child-uid" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources) { + repoResources.On("EnsureFolderTreeExists", mock.Anything, "feature/branch", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { + expectedFolders := []resources.Folder{ + {ID: "parent-folder", Path: "parent-folder"}, + {ID: "child-folder", Path: "parent-folder/child-folder"}, + } + + if tree.Count() != len(expectedFolders) { + return false + } + + for _, folder := range expectedFolders { + dir, ok := tree.DirPath(folder.ID, "") + if !ok || dir.Path != folder.Path { + return false + } + } + + return true + }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { + // Parent folder should be processed first + require.NoError(t, fn(resources.Folder{ID: "parent-uid", Path: "grafana/parent-folder"}, true, nil)) + // Then child folder with nested path + require.NoError(t, fn(resources.Folder{ID: "child-uid", Path: "grafana/parent-folder/child-folder"}, true, nil)) + return true + })).Return(nil) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: "FolderList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: resources.FolderResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.FolderResource: listGVK.Kind, + }) + fakeFolderClient := fakeDynamicClient.Resource(resources.FolderResource) + fakeDynamicClient.PrependReactor("list", "folders", tt.reactorFunc) + mockProgress := jobs.NewMockJobProgressRecorder(t) + tt.setupProgress(mockProgress) + + repoResources := resources.NewMockRepositoryResources(t) + tt.setupResources(repoResources) + + err := ExportFolders(context.Background(), "test-repo", v0alpha1.ExportJobOptions{ + Path: "grafana", + Branch: "feature/branch", + }, fakeFolderClient, repoResources, mockProgress) + + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + } + + repoResources.AssertExpectations(t) + mockProgress.AssertExpectations(t) + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/export/mock_export_fn.go b/pkg/registry/apis/provisioning/jobs/export/mock_export_fn.go new file mode 100644 index 00000000000..7e7a405055a --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/mock_export_fn.go @@ -0,0 +1,95 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package export + +import ( + context "context" + + jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + dynamic "k8s.io/client-go/dynamic" + + mock "github.com/stretchr/testify/mock" + + resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" +) + +// MockExportFn is an autogenerated mock type for the ExportFn type +type MockExportFn struct { + mock.Mock +} + +type MockExportFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockExportFn) EXPECT() *MockExportFn_Expecter { + return &MockExportFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repoName, options, clients, repositoryResources, folderClient, progress +func (_m *MockExportFn) Execute(ctx context.Context, repoName string, options v0alpha1.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, folderClient dynamic.ResourceInterface, progress jobs.JobProgressRecorder) error { + ret := _m.Called(ctx, repoName, options, clients, repositoryResources, folderClient, progress) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, v0alpha1.ExportJobOptions, resources.ResourceClients, resources.RepositoryResources, dynamic.ResourceInterface, jobs.JobProgressRecorder) error); ok { + r0 = rf(ctx, repoName, options, clients, repositoryResources, folderClient, progress) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockExportFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockExportFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repoName string +// - options v0alpha1.ExportJobOptions +// - clients resources.ResourceClients +// - repositoryResources resources.RepositoryResources +// - folderClient dynamic.ResourceInterface +// - progress jobs.JobProgressRecorder +func (_e *MockExportFn_Expecter) Execute(ctx interface{}, repoName interface{}, options interface{}, clients interface{}, repositoryResources interface{}, folderClient interface{}, progress interface{}) *MockExportFn_Execute_Call { + return &MockExportFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repoName, options, clients, repositoryResources, folderClient, progress)} +} + +func (_c *MockExportFn_Execute_Call) Run(run func(ctx context.Context, repoName string, options v0alpha1.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, folderClient dynamic.ResourceInterface, progress jobs.JobProgressRecorder)) *MockExportFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(v0alpha1.ExportJobOptions), args[3].(resources.ResourceClients), args[4].(resources.RepositoryResources), args[5].(dynamic.ResourceInterface), args[6].(jobs.JobProgressRecorder)) + }) + return _c +} + +func (_c *MockExportFn_Execute_Call) Return(_a0 error) *MockExportFn_Execute_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockExportFn_Execute_Call) RunAndReturn(run func(context.Context, string, v0alpha1.ExportJobOptions, resources.ResourceClients, resources.RepositoryResources, dynamic.ResourceInterface, jobs.JobProgressRecorder) error) *MockExportFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockExportFn creates a new instance of MockExportFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockExportFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockExportFn { + mock := &MockExportFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/export/mock_wrap_with_clone_fn.go b/pkg/registry/apis/provisioning/jobs/export/mock_wrap_with_clone_fn.go new file mode 100644 index 00000000000..bc690475577 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/mock_wrap_with_clone_fn.go @@ -0,0 +1,87 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package export + +import ( + context "context" + + repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + mock "github.com/stretchr/testify/mock" +) + +// MockWrapWithCloneFn is an autogenerated mock type for the WrapWithCloneFn type +type MockWrapWithCloneFn struct { + mock.Mock +} + +type MockWrapWithCloneFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockWrapWithCloneFn) EXPECT() *MockWrapWithCloneFn_Expecter { + return &MockWrapWithCloneFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repo, cloneOptions, pushOptions, fn +func (_m *MockWrapWithCloneFn) Execute(ctx context.Context, repo repository.Repository, cloneOptions repository.CloneOptions, pushOptions repository.PushOptions, fn func(repository.Repository, bool) error) error { + ret := _m.Called(ctx, repo, cloneOptions, pushOptions, fn) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, repository.Repository, repository.CloneOptions, repository.PushOptions, func(repository.Repository, bool) error) error); ok { + r0 = rf(ctx, repo, cloneOptions, pushOptions, fn) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockWrapWithCloneFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockWrapWithCloneFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repo repository.Repository +// - cloneOptions repository.CloneOptions +// - pushOptions repository.PushOptions +// - fn func(repository.Repository , bool) error +func (_e *MockWrapWithCloneFn_Expecter) Execute(ctx interface{}, repo interface{}, cloneOptions interface{}, pushOptions interface{}, fn interface{}) *MockWrapWithCloneFn_Execute_Call { + return &MockWrapWithCloneFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, cloneOptions, pushOptions, fn)} +} + +func (_c *MockWrapWithCloneFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Repository, cloneOptions repository.CloneOptions, pushOptions repository.PushOptions, fn func(repository.Repository, bool) error)) *MockWrapWithCloneFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(repository.Repository), args[2].(repository.CloneOptions), args[3].(repository.PushOptions), args[4].(func(repository.Repository, bool) error)) + }) + return _c +} + +func (_c *MockWrapWithCloneFn_Execute_Call) Return(_a0 error) *MockWrapWithCloneFn_Execute_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockWrapWithCloneFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Repository, repository.CloneOptions, repository.PushOptions, func(repository.Repository, bool) error) error) *MockWrapWithCloneFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockWrapWithCloneFn creates a new instance of MockWrapWithCloneFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockWrapWithCloneFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockWrapWithCloneFn { + mock := &MockWrapWithCloneFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/export/resources.go b/pkg/registry/apis/provisioning/jobs/export/resources.go new file mode 100644 index 00000000000..7d57a2b1039 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/resources.go @@ -0,0 +1,68 @@ +package export + +import ( + "context" + "errors" + "fmt" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" +) + +func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + progress.SetMessage(ctx, "start resource export") + for _, kind := range resources.SupportedProvisioningResources { + // skip from folders as we do them first... so only dashboards + if kind == resources.FolderResource { + continue + } + + progress.SetMessage(ctx, fmt.Sprintf("export %s", kind.Resource)) + client, _, err := clients.ForResource(kind) + if err != nil { + return fmt.Errorf("get client for %s: %w", kind.Resource, err) + } + + if err := exportResource(ctx, options, client, repositoryResources, progress); err != nil { + return fmt.Errorf("export %s: %w", kind.Resource, err) + } + } + + return nil +} + +func exportResource(ctx context.Context, options provisioning.ExportJobOptions, client dynamic.ResourceInterface, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + return resources.ForEach(ctx, client, func(item *unstructured.Unstructured) error { + fileName, err := repositoryResources.CreateResourceFileFromObject(ctx, item, resources.WriteOptions{ + Path: options.Path, + Ref: options.Branch, + }) + + gvk := item.GroupVersionKind() + result := jobs.JobResourceResult{ + Name: item.GetName(), + Resource: gvk.Kind, + Group: gvk.Group, + Action: repository.FileActionCreated, + Path: fileName, + } + + if errors.Is(err, resources.ErrAlreadyInRepository) { + result.Action = repository.FileActionIgnored + } else if err != nil { + result.Action = repository.FileActionIgnored + result.Error = err + } + + progress.Record(ctx, result) + if err := progress.TooManyErrors(); err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/registry/apis/provisioning/jobs/export/resources_test.go b/pkg/registry/apis/provisioning/jobs/export/resources_test.go new file mode 100644 index 00000000000..a8346ad1429 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/export/resources_test.go @@ -0,0 +1,311 @@ +package export + +import ( + "context" + "fmt" + "testing" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + mock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8testing "k8s.io/client-go/testing" +) + +func TestExportResources(t *testing.T) { + tests := []struct { + name string + reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) + expectedError string + setupProgress func(progress *jobs.MockJobProgressRecorder) + setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) + }{ + { + name: "successful dashboard export", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + // Return dashboard list + return true, &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "DashboardList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard-1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard-2", + }, + }, + }, + }, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-1" && result.Action == repository.FileActionCreated + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-2" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-1" + }), options).Return("dashboard-1.json", nil) + + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-2" + }), options).Return("dashboard-2.json", nil) + }, + }, + { + name: "client error", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("shouldn't happen") + }, + expectedError: "get client for dashboards: didn't work", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, fmt.Errorf("didn't work")) + }, + }, + { + name: "dashboard list error", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("failed to list dashboards") + }, + expectedError: "export dashboards: error executing list: failed to list dashboards", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + }, + }, + { + name: "dashboard export with errors", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "DashboardList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard-1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard-2", + }, + }, + }, + }, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-1" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "failed to export dashboard" + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-2" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-1" + }), options).Return("", fmt.Errorf("failed to export dashboard")) + + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-2" + }), options).Return("dashboard-2.json", nil) + }, + }, + { + name: "dashboard export too many errors", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "DashboardList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard-1", + }, + }, + }, + }, nil + }, + expectedError: "export dashboards: too many errors encountered", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-1" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "failed to export dashboard" + })).Return() + progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered")) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "dashboard-1" + }), options).Return("", fmt.Errorf("failed to export dashboard")) + }, + }, + { + name: "ignores existing dashboards", + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "DashboardList", + }, + Items: []metav1.PartialObjectMetadata{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: resources.DashboardResource.GroupVersion().String(), + Kind: "Dashboard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-dashboard", + }, + }, + }, + }, nil + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "existing-dashboard" && result.Action == repository.FileActionIgnored + })).Return() + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + + // Return true to indicate the file already exists, and provide the updated path + repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + return obj.GetName() == "existing-dashboard" + }), options).Return("", resources.ErrAlreadyInRepository) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: "DashboardList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: resources.DashboardResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.DashboardResource: listGVK.Kind, + }) + + resourceClients := resources.NewMockResourceClients(t) + fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc) + + mockProgress := jobs.NewMockJobProgressRecorder(t) + tt.setupProgress(mockProgress) + + repoResources := resources.NewMockRepositoryResources(t) + tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVK) + + options := v0alpha1.ExportJobOptions{ + Path: "grafana", + Branch: "feature/branch", + } + + err := ExportResources(context.Background(), options, resourceClients, repoResources, mockProgress) + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + } + + mockProgress.AssertExpectations(t) + repoResources.AssertExpectations(t) + resourceClients.AssertExpectations(t) + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/export/worker.go b/pkg/registry/apis/provisioning/jobs/export/worker.go index 420a0190ed3..73e4eaa6c2e 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker.go @@ -7,26 +7,37 @@ import ( "os" "time" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "k8s.io/client-go/dynamic" ) +//go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter +type ExportFn func(ctx context.Context, repoName string, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, folderClient dynamic.ResourceInterface, progress jobs.JobProgressRecorder) error + +//go:generate mockery --name WrapWithCloneFn --structname MockWrapWithCloneFn --inpackage --filename mock_wrap_with_clone_fn.go --with-expecter +type WrapWithCloneFn func(ctx context.Context, repo repository.Repository, cloneOptions repository.CloneOptions, pushOptions repository.PushOptions, fn func(repo repository.Repository, cloned bool) error) error + type ExportWorker struct { clientFactory resources.ClientFactory repositoryResources resources.RepositoryResourcesFactory + exportFn ExportFn + wrapWithCloneFn WrapWithCloneFn } func NewExportWorker( clientFactory resources.ClientFactory, repositoryResources resources.RepositoryResourcesFactory, + exportFn ExportFn, + wrapWithCloneFn WrapWithCloneFn, ) *ExportWorker { return &ExportWorker{ clientFactory: clientFactory, repositoryResources: repositoryResources, + exportFn: exportFn, + wrapWithCloneFn: wrapWithCloneFn, } } @@ -52,6 +63,11 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, PushOnWrites: false, BeforeFn: func() error { progress.SetMessage(ctx, "clone target") + // :( the branch is now baked into the repo + if options.Branch != "" { + return fmt.Errorf("branch is not supported for clonable repositories") + } + return nil }, } @@ -65,20 +81,12 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, }, } - fn := func(repo repository.Repository, cloned bool) error { - if cloned { - options.Branch = "" // :( the branch is now baked into the repo - } - - // Load and write all folders - // FIXME: we load the entire tree in memory - progress.SetMessage(ctx, "read folder tree from API server") + fn := func(repo repository.Repository, _ bool) error { clients, err := r.clientFactory.Clients(ctx, cfg.Namespace) if err != nil { return fmt.Errorf("create clients: %w", err) } - tree := resources.NewEmptyFolderTree() folderClient, err := clients.Folder() if err != nil { return fmt.Errorf("create folder client: %w", err) @@ -94,88 +102,8 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, return fmt.Errorf("create repository resource client: %w", err) } - if err := resources.ForEach(ctx, folderClient, func(item *unstructured.Unstructured) error { - if tree.Count() >= resources.MaxNumberOfFolders { - return errors.New("too many folders") - } - - return tree.AddUnstructured(item, cfg.Name) - }); err != nil { - return fmt.Errorf("load folder tree: %w", err) - } - - progress.SetMessage(ctx, "write folders to repository") - err = repositoryResources.EnsureFolderTreeExists(ctx, options.Branch, options.Path, tree, func(folder resources.Folder, created bool, err error) error { - result := jobs.JobResourceResult{ - Action: repository.FileActionCreated, - Name: folder.ID, - Resource: resources.FolderResource.Resource, - Group: resources.FolderResource.Group, - Path: folder.Path, - Error: err, - } - - if !created { - result.Action = repository.FileActionIgnored - } - - progress.Record(ctx, result) - if err := progress.TooManyErrors(); err != nil { - return err - } - - return nil - }) - - if err != nil { - return fmt.Errorf("write folders to repository: %w", err) - } - - progress.SetMessage(ctx, "start resource export") - for _, kind := range resources.SupportedProvisioningResources { - // skip from folders as we do them first... so only dashboards - if kind == resources.FolderResource { - continue - } - - progress.SetMessage(ctx, fmt.Sprintf("export %s", kind.Resource)) - client, _, err := clients.ForResource(kind) - if err != nil { - return err - } - - if err := resources.ForEach(ctx, client, func(item *unstructured.Unstructured) error { - result := jobs.JobResourceResult{ - Name: item.GetName(), - Resource: kind.Resource, - Group: kind.Group, - Action: repository.FileActionCreated, - } - - fileName, err := repositoryResources.CreateResourceFileFromObject(ctx, item, resources.WriteOptions{ - Path: options.Path, - Ref: options.Branch, - }) - if errors.Is(err, resources.ErrAlreadyInRepository) { - result.Action = repository.FileActionIgnored - } else if err != nil { - result.Action = repository.FileActionIgnored - result.Error = err - } - result.Path = fileName - progress.Record(ctx, result) - - if err := progress.TooManyErrors(); err != nil { - return err - } - return nil - }); err != nil { - return fmt.Errorf("export %s: %w", kind.Resource, err) - } - } - - return nil + return r.exportFn(ctx, cfg.Name, *options, clients, repositoryResources, folderClient, progress) } - return repository.WrapWithCloneAndPushIfPossible(ctx, repo, cloneOptions, pushOptions, fn) + return r.wrapWithCloneFn(ctx, repo, cloneOptions, pushOptions, fn) } diff --git a/pkg/registry/apis/provisioning/jobs/export/worker_test.go b/pkg/registry/apis/provisioning/jobs/export/worker_test.go index 193a3942c50..d5ad42c509b 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "testing" "time" @@ -11,14 +12,10 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" - "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/assert" + mock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - dynamicfake "k8s.io/client-go/dynamic/fake" - k8testing "k8s.io/client-go/testing" ) func TestExportWorker_IsSupported(t *testing.T) { @@ -58,7 +55,7 @@ func TestExportWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := NewExportWorker(nil, nil) + r := NewExportWorker(nil, nil, nil, nil) got := r.IsSupported(context.Background(), tt.job) require.Equal(t, tt.want, got) }) @@ -72,7 +69,7 @@ func TestExportWorker_ProcessNoExportSettings(t *testing.T) { }, } - r := NewExportWorker(nil, nil) + r := NewExportWorker(nil, nil, nil, nil) err := r.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing export settings") } @@ -95,7 +92,7 @@ func TestExportWorker_ProcessWriteNotAllowed(t *testing.T) { }, }) - r := NewExportWorker(nil, nil) + r := NewExportWorker(nil, nil, nil, nil) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository is read only") } @@ -118,7 +115,7 @@ func TestExportWorker_ProcessBranchNotAllowedForLocal(t *testing.T) { }, }) - r := NewExportWorker(nil, nil) + r := NewExportWorker(nil, nil, nil, nil) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository does not support the branch workflow") } @@ -143,11 +140,15 @@ func TestExportWorker_ProcessFailedToCreateClients(t *testing.T) { }) mockClients := resources.NewMockClientFactory(t) - mockClients.On("Clients", context.Background(), "test-namespace").Return(nil, errors.New("failed to create clients")) - r := NewExportWorker(mockClients, nil) + mockClients.On("Clients", context.Background(), "test-namespace").Return(nil, errors.New("failed to create clients")) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + return fn(repo, true) + }) + + r := NewExportWorker(mockClients, nil, nil, mockCloneFn.Execute) mockProgress := jobs.NewMockJobProgressRecorder(t) - mockProgress.On("SetMessage", context.Background(), "read folder tree from API server").Return() err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "create clients: failed to create clients") @@ -177,9 +178,13 @@ func TestExportWorker_ProcessNotReaderWriter(t *testing.T) { mockClients.On("Clients", context.Background(), "test-namespace").Return(resourceClients, nil) resourceClients.On("Folder").Return(nil, nil) mockProgress := jobs.NewMockJobProgressRecorder(t) - mockProgress.On("SetMessage", context.Background(), "read folder tree from API server").Return() - r := NewExportWorker(mockClients, nil) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + return fn(repo, true) + }) + + r := NewExportWorker(mockClients, nil, nil, mockCloneFn.Execute) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export job submitted targeting repository that is not a ReaderWriter") } @@ -209,9 +214,11 @@ func TestExportWorker_ProcessFolderClientError(t *testing.T) { resourceClients.On("Folder").Return(nil, fmt.Errorf("failed to create folder client")) mockProgress := jobs.NewMockJobProgressRecorder(t) - mockProgress.On("SetMessage", context.Background(), "read folder tree from API server").Return() - - r := NewExportWorker(mockClients, nil) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + return fn(repo, true) + }) + r := NewExportWorker(mockClients, nil, nil, mockCloneFn.Execute) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "create folder client: failed to create folder client") } @@ -244,1112 +251,170 @@ func TestExportWorker_ProcessRepositoryResourcesError(t *testing.T) { mockRepoResources.On("Client", context.Background(), mockRepo).Return(nil, fmt.Errorf("failed to create repository resources client")) mockProgress := jobs.NewMockJobProgressRecorder(t) - mockProgress.On("SetMessage", context.Background(), "read folder tree from API server").Return() - - r := NewExportWorker(mockClients, mockRepoResources) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + return fn(repo, true) + }) + r := NewExportWorker(mockClients, mockRepoResources, nil, mockCloneFn.Execute) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "create repository resource client: failed to create repository resources client") } -func TestExportWorker_ProcessFolders(t *testing.T) { - tests := []struct { - name string - reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) - expectedError string - setupProgress func(progress *jobs.MockJobProgressRecorder) - setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) - verifyMocks func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) - }{ - { - name: "list folders error", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("failed to list folders") - }, - expectedError: "load folder tree: error executing list: failed to list folders", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, mock.Anything).Return() - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - }, - }, - { - name: "too many folders", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - list := &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - Items: make([]metav1.PartialObjectMetadata, resources.MaxNumberOfFolders+1), - } - for i := 0; i <= resources.MaxNumberOfFolders; i++ { - list.Items[i] = metav1.PartialObjectMetadata{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("folder-%d", i), - }, - } - } - return true, list, nil - }, - expectedError: "load folder tree: too many folders", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, mock.Anything).Return() - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "ensure folder tree error", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - // Return empty list to get past the folder loading - return true, &metav1.PartialObjectMetadataList{}, nil - }, - expectedError: "write folders to repository: failed to ensure folder tree", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, mock.Anything).Return() - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.Anything, mock.Anything).Return(fmt.Errorf("failed to ensure folder tree")) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "successful folder migration", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.DashboardResource { - // Return empty dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "FolderList", - }, - }, nil - } - - list := &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "folder-1", - Annotations: map[string]string{ - "folder.grafana.app/uid": "folder-1-uid", - }, - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "folder-2", - Annotations: map[string]string{ - "folder.grafana.app/uid": "folder-2-uid", - }, - }, - }, - }, - } - return true, list, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "folder-1-uid" && result.Action == repository.FileActionCreated - })).Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "folder-2-uid" && result.Action == repository.FileActionCreated - })).Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("TooManyErrors").Return(nil) - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 2 - }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { - require.NoError(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, true, nil)) - require.NoError(t, fn(resources.Folder{ID: "folder-2-uid", Path: "grafana/folder-2"}, true, nil)) - - return true - })).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "successful folder migration with resource export errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.DashboardResource { - // Return empty dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "FolderList", - }, - }, nil - } - - list := &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "folder-1", - Annotations: map[string]string{ - "folder.grafana.app/uid": "folder-1-uid", - }, - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "folder-2", - Annotations: map[string]string{ - "folder.grafana.app/uid": "folder-2-uid", - }, - }, - }, - }, - } - return true, list, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "folder-1-uid" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "didn't work" - })).Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "folder-2-uid" && result.Action == repository.FileActionCreated - })).Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("TooManyErrors").Return(nil) - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 2 - }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { - require.NoError(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, false, errors.New("didn't work"))) - require.NoError(t, fn(resources.Folder{ID: "folder-2-uid", Path: "grafana/folder-2"}, true, nil)) - - return true - })).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "too many errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - list := &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "folder-1", - Annotations: map[string]string{ - "folder.grafana.app/uid": "folder-1-uid", - }, - }, - }, - }, - } - return true, list, nil - }, - expectedError: "write folders to repository: too many errors encountered", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("Record", mock.Anything, mock.Anything).Return() - progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered")) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 1 - }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { - require.Error(t, fn(resources.Folder{ID: "folder-1-uid", Path: "grafana/folder-1"}, true, nil), "too many errors encountered") - return true - })).Return(fmt.Errorf("too many errors encountered")) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "successful nested folder migration", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.DashboardResource { - // Return empty dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "FolderList", - }, - }, nil - } - - list := &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "parent-folder", - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "Folder", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "child-folder", - Annotations: map[string]string{ - "grafana.app/folder": "parent-folder", - }, - }, - }, - }, - } - return true, list, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "parent-uid" && result.Action == repository.FileActionCreated - })).Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "child-uid" && result.Action == repository.FileActionCreated - })).Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("TooManyErrors").Return(nil) - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - expectedFolders := []resources.Folder{ - {ID: "parent-folder", Path: "parent-folder"}, - {ID: "child-folder", Path: "parent-folder/child-folder"}, - } - - if tree.Count() != len(expectedFolders) { - return false - } - - for _, folder := range expectedFolders { - dir, ok := tree.DirPath(folder.ID, "") - if !ok || dir.Path != folder.Path { - return false - } - } - - return true - }), mock.MatchedBy(func(fn func(folder resources.Folder, created bool, err error) error) bool { - // Parent folder should be processed first - require.NoError(t, fn(resources.Folder{ID: "parent-uid", Path: "grafana/parent-folder"}, true, nil)) - // Then child folder with nested path - require.NoError(t, fn(resources.Folder{ID: "child-uid", Path: "grafana/parent-folder/child-folder"}, true, nil)) - return true - })).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, +func TestExportWorker_ProcessCloneAndPushOptions(t *testing.T) { + job := v0alpha1.Job{ + Spec: v0alpha1.JobSpec{ + Action: v0alpha1.JobActionPush, + Push: &v0alpha1.ExportJobOptions{}, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - job := v0alpha1.Job{ - Spec: v0alpha1.JobSpec{ - Action: v0alpha1.JobActionPush, - Push: &v0alpha1.ExportJobOptions{ - Path: "grafana", - }, - }, - } + mockRepo := repository.NewMockRepository(t) + mockRepo.On("Config").Return(&v0alpha1.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Spec: v0alpha1.RepositorySpec{ + Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, + }, + }) - mockRepo := repository.NewMockRepository(t) - mockRepo.On("Config").Return(&v0alpha1.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo", - Namespace: "test-namespace", - }, - Spec: v0alpha1.RepositorySpec{ - Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, - }, - }) + mockProgress := jobs.NewMockJobProgressRecorder(t) + // Verify progress messages are set + mockProgress.On("SetMessage", mock.Anything, "clone target").Return() + mockProgress.On("SetMessage", mock.Anything, "push changes").Return() - scheme := runtime.NewScheme() - require.NoError(t, metav1.AddMetaToScheme(scheme)) - listGVK := schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: "FolderList", - } - listGVKDashboard := schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: "DashboardList", - } + mockClients := resources.NewMockClientFactory(t) + mockResourceClients := resources.NewMockResourceClients(t) + mockClients.On("Clients", mock.Anything, "test-namespace").Return(mockResourceClients, nil) + mockResourceClients.On("Folder").Return(nil, nil) - scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(listGVKDashboard, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: resources.FolderResource.Resource, - }, &metav1.PartialObjectMetadata{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: resources.DashboardResource.Resource, - }, &metav1.PartialObjectMetadata{}) + mockRepoResources := resources.NewMockRepositoryResourcesFactory(t) + mockRepoResourcesClient := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Client", mock.Anything, mock.Anything).Return(mockRepoResourcesClient, nil) - fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - resources.FolderResource: listGVK.Kind, - resources.DashboardResource: listGVKDashboard.Kind, - }) - fakeFolderClient := fakeDynamicClient.Resource(resources.FolderResource) + mockExportFn := NewMockExportFn(t) + mockExportFn.On("Execute", mock.Anything, "test-repo", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - resourceClients := resources.NewMockResourceClients(t) - resourceClients.On("Folder").Return(fakeFolderClient, nil) + mockCloneFn := NewMockWrapWithCloneFn(t) + // Verify clone and push options + mockCloneFn.On("Execute", mock.Anything, mockRepo, mock.MatchedBy(func(opts repository.CloneOptions) bool { + return opts.Timeout == 10*time.Minute && !opts.PushOnWrites && opts.BeforeFn != nil + }), mock.MatchedBy(func(opts repository.PushOptions) bool { + return opts.Timeout == 10*time.Minute && opts.Progress == os.Stdout && opts.BeforeFn != nil + }), mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + // Execute both BeforeFn functions to verify progress messages + assert.NoError(t, cloneOpts.BeforeFn()) + assert.NoError(t, pushOpts.BeforeFn()) - mockClientFactory := resources.NewMockClientFactory(t) - mockClientFactory.On("Clients", context.Background(), "test-namespace").Return(resourceClients, nil) + return fn(repo, true) + }) - fakeDynamicClient.PrependReactor("list", "folders", tt.reactorFunc) - fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc) - - mockProgress := jobs.NewMockJobProgressRecorder(t) - tt.setupProgress(mockProgress) - - repoResources := resources.NewMockRepositoryResources(t) - tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVKDashboard) - mockRepoResources := resources.NewMockRepositoryResourcesFactory(t) - mockRepoResources.On("Client", mock.Anything, mockRepo).Return(repoResources, nil) - - r := NewExportWorker(mockClientFactory, mockRepoResources) - err := r.Process(context.Background(), mockRepo, job, mockProgress) - - if tt.expectedError != "" { - require.EqualError(t, err, tt.expectedError) - } else { - require.NoError(t, err) - } - - tt.verifyMocks(t, mockProgress, repoResources) - }) - } + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockCloneFn.Execute) + err := r.Process(context.Background(), mockRepo, job, mockProgress) + require.NoError(t, err) } -func TestExportWorker_ProcessDashboards(t *testing.T) { - tests := []struct { - name string - reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) - expectedError string - setupProgress func(progress *jobs.MockJobProgressRecorder) - setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) - verifyMocks func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) - }{ - { - name: "successful dashboard export", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - // Return empty folder list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - }, nil - } - // Return dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-2", - }, - }, - }, - }, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "dashboard-1" && result.Action == repository.FileActionCreated - })).Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "dashboard-2" && result.Action == repository.FileActionCreated - })).Return() - progress.On("TooManyErrors").Return(nil) - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - - options := resources.WriteOptions{ - Path: "grafana", - // TODO: add tests for branch - Ref: "", - } - - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "dashboard-1" - }), options).Return("dashboard-1.json", nil) - - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "dashboard-2" - }), options).Return("dashboard-2.json", nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "dashboard list error", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - // Return empty folder list - return true, &metav1.PartialObjectMetadataList{}, nil - } - // Return error for dashboard list - return true, nil, fmt.Errorf("failed to list dashboards") - }, - expectedError: "export dashboards: error executing list: failed to list dashboards", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "dashboard export with errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - return true, &metav1.PartialObjectMetadataList{}, nil - } - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-2", - }, - }, - }, - }, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "dashboard-1" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "failed to export dashboard" - })).Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "dashboard-2" && result.Action == repository.FileActionCreated - })).Return() - progress.On("TooManyErrors").Return(nil) - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - - options := resources.WriteOptions{ - Path: "grafana", - Ref: "", - } - - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "dashboard-1" - }), options).Return("", fmt.Errorf("failed to export dashboard")) - - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "dashboard-2" - }), options).Return("dashboard-2.json", nil) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "dashboard export too many errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - return true, &metav1.PartialObjectMetadataList{}, nil - } - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, - }, - }, - }, nil - }, - expectedError: "export dashboards: too many errors encountered", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "dashboard-1" && result.Action == repository.FileActionIgnored && result.Error != nil && result.Error.Error() == "failed to export dashboard" - })).Return() - progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered")) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - - options := resources.WriteOptions{ - Path: "grafana", - Ref: "", - } - - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "dashboard-1" - }), options).Return("", fmt.Errorf("failed to export dashboard")) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, - }, - { - name: "ignores existing dashboards", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - return true, &metav1.PartialObjectMetadataList{}, nil - } - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "existing-dashboard", - }, - }, - }, - }, nil - }, - expectedError: "", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { - return result.Name == "existing-dashboard" && result.Action == repository.FileActionIgnored - })).Return() - progress.On("TooManyErrors").Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - - options := resources.WriteOptions{ - Path: "grafana", - Ref: "", - } - - // Return true to indicate the file already exists, and provide the updated path - repoResources.On("CreateResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { - return obj.GetName() == "existing-dashboard" - }), options).Return("", resources.ErrAlreadyInRepository) - }, - verifyMocks: func(t *testing.T, progress *jobs.MockJobProgressRecorder, repoResources *resources.MockRepositoryResources) { - progress.AssertExpectations(t) - repoResources.AssertExpectations(t) - }, +func TestExportWorker_ProcessExportFnError(t *testing.T) { + job := v0alpha1.Job{ + Spec: v0alpha1.JobSpec{ + Action: v0alpha1.JobActionPush, + Push: &v0alpha1.ExportJobOptions{}, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - job := v0alpha1.Job{ - Spec: v0alpha1.JobSpec{ - Action: v0alpha1.JobActionPush, - Push: &v0alpha1.ExportJobOptions{ - Path: "grafana", - }, - }, - } + mockRepo := repository.NewMockRepository(t) + mockRepo.On("Config").Return(&v0alpha1.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Spec: v0alpha1.RepositorySpec{ + Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, + }, + }) - mockRepo := repository.NewMockRepository(t) - mockRepo.On("Config").Return(&v0alpha1.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo", - Namespace: "test-namespace", - }, - Spec: v0alpha1.RepositorySpec{ - Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, - }, - }) + mockProgress := jobs.NewMockJobProgressRecorder(t) + mockClients := resources.NewMockClientFactory(t) + mockResourceClients := resources.NewMockResourceClients(t) + mockClients.On("Clients", mock.Anything, "test-namespace").Return(mockResourceClients, nil) + mockResourceClients.On("Folder").Return(nil, nil) - scheme := runtime.NewScheme() - require.NoError(t, metav1.AddMetaToScheme(scheme)) - listGVK := schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: "FolderList", - } - listGVKDashboard := schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: "DashboardList", - } + mockRepoResources := resources.NewMockRepositoryResourcesFactory(t) + mockRepoResourcesClient := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Client", mock.Anything, mock.Anything).Return(mockRepoResourcesClient, nil) - scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(listGVKDashboard, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: resources.FolderResource.Resource, - }, &metav1.PartialObjectMetadata{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: resources.DashboardResource.Resource, - }, &metav1.PartialObjectMetadata{}) + mockExportFn := NewMockExportFn(t) + mockExportFn.On("Execute", mock.Anything, "test-repo", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("export failed")) - fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - resources.FolderResource: listGVK.Kind, - resources.DashboardResource: listGVKDashboard.Kind, - }) - fakeFolderClient := fakeDynamicClient.Resource(resources.FolderResource) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + return fn(repo, true) + }) - resourceClients := resources.NewMockResourceClients(t) - resourceClients.On("Folder").Return(fakeFolderClient, nil) - - mockClientFactory := resources.NewMockClientFactory(t) - mockClientFactory.On("Clients", context.Background(), "test-namespace").Return(resourceClients, nil) - - fakeDynamicClient.PrependReactor("list", "folders", tt.reactorFunc) - fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc) - - mockProgress := jobs.NewMockJobProgressRecorder(t) - tt.setupProgress(mockProgress) - - repoResources := resources.NewMockRepositoryResources(t) - tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVKDashboard) - mockRepoResources := resources.NewMockRepositoryResourcesFactory(t) - mockRepoResources.On("Client", mock.Anything, mockRepo).Return(repoResources, nil) - - r := NewExportWorker(mockClientFactory, mockRepoResources) - err := r.Process(context.Background(), mockRepo, job, mockProgress) - - if tt.expectedError != "" { - require.EqualError(t, err, tt.expectedError) - } else { - require.NoError(t, err) - } - - tt.verifyMocks(t, mockProgress, repoResources) - }) - } + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockCloneFn.Execute) + err := r.Process(context.Background(), mockRepo, job, mockProgress) + require.EqualError(t, err, "export failed") } -type MockClonableRepository struct { - *repository.MockClonableRepository - *repository.MockClonedRepository -} - -func TestExportWorker_ClonableRepository(t *testing.T) { - tests := []struct { - name string - createRepo func(t *testing.T) *MockClonableRepository - reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) - setupRepo func(repo *repository.MockClonedRepository) - setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) - setupProgress func(progress *jobs.MockJobProgressRecorder) - expectedError string - }{ - { - name: "successful clone and push", - createRepo: func(t *testing.T) *MockClonableRepository { - cloned := repository.NewMockClonedRepository(t) - clonable := repository.NewMockClonableRepository(t) - clonable.On("Clone", mock.Anything, mock.MatchedBy(func(opts repository.CloneOptions) bool { - if opts.PushOnWrites || opts.Timeout != 10*time.Minute { - return false - } - - if opts.BeforeFn != nil { - require.NoError(t, opts.BeforeFn()) - } - - return true - })).Return(cloned, nil) - - return &MockClonableRepository{ - MockClonedRepository: cloned, - MockClonableRepository: clonable, - } - }, - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - if action.GetResource() == resources.FolderResource { - // Return empty folder list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.FolderResource.GroupVersion().String(), - Kind: "FolderList", - }, - }, nil - } - // Return empty dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - }, nil - }, - setupRepo: func(repo *repository.MockClonedRepository) { - repo.On("Config").Return(&v0alpha1.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo", - Namespace: "test-namespace", - }, - Spec: v0alpha1.RepositorySpec{ - Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, - }, - }) - repo.On("Push", mock.Anything, mock.MatchedBy(func(opts repository.PushOptions) bool { - if opts.Timeout != 10*time.Minute { - return false - } - - if opts.BeforeFn != nil { - require.NoError(t, opts.BeforeFn()) - } - - return true - })).Return(nil) - repo.On("Remove", mock.Anything).Return(nil) - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - repoResources.On("EnsureFolderTreeExists", mock.Anything, "", "grafana", mock.MatchedBy(func(tree resources.FolderTree) bool { - return tree.Count() == 0 - }), mock.Anything).Return(nil) - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) - }, - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "clone target").Return() - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - progress.On("SetMessage", mock.Anything, "write folders to repository").Return() - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - progress.On("SetMessage", mock.Anything, "push changes").Return() - }, - expectedError: "", - }, - { - name: "clone failure", - createRepo: func(t *testing.T) *MockClonableRepository { - cloned := repository.NewMockClonedRepository(t) - clonable := repository.NewMockClonableRepository(t) - clonable.On("Clone", mock.Anything, mock.MatchedBy(func(opts repository.CloneOptions) bool { - if opts.BeforeFn != nil { - require.NoError(t, opts.BeforeFn()) - } - - return true - })).Return(nil, fmt.Errorf("failed to clone repository")) - - return &MockClonableRepository{ - MockClonedRepository: cloned, - MockClonableRepository: clonable, - } - }, - setupRepo: func(repo *repository.MockClonedRepository) { - repo.On("Config").Return(&v0alpha1.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo", - Namespace: "test-namespace", - }, - Spec: v0alpha1.RepositorySpec{ - Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, - }, - }) - }, - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "clone target").Return() - }, - expectedError: "clone repository: failed to clone repository", - }, - { - name: "any other failure cleans up the cloned repository", - createRepo: func(t *testing.T) *MockClonableRepository { - cloned := repository.NewMockClonedRepository(t) - clonable := repository.NewMockClonableRepository(t) - clonable.On("Clone", mock.Anything, mock.MatchedBy(func(opts repository.CloneOptions) bool { - if opts.BeforeFn != nil { - require.NoError(t, opts.BeforeFn()) - } - - return true - })).Return(cloned, nil) - - return &MockClonableRepository{ - MockClonedRepository: cloned, - MockClonableRepository: clonable, - } - }, - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("some error") - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - // nothing special to do here - }, - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "clone target").Return() - progress.On("SetMessage", mock.Anything, "read folder tree from API server").Return() - }, - setupRepo: func(repo *repository.MockClonedRepository) { - repo.On("Config").Return(&v0alpha1.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-repo", - Namespace: "test-namespace", - }, - Spec: v0alpha1.RepositorySpec{ - Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, - }, - }) - repo.On("Remove", mock.Anything).Maybe().Return(nil) - }, - expectedError: "load folder tree: error executing list: some error", +func TestExportWorker_ProcessWrapWithCloneFnError(t *testing.T) { + job := v0alpha1.Job{ + Spec: v0alpha1.JobSpec{ + Action: v0alpha1.JobActionPush, + Push: &v0alpha1.ExportJobOptions{}, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - job := v0alpha1.Job{ - Spec: v0alpha1.JobSpec{ - Action: v0alpha1.JobActionPush, - Push: &v0alpha1.ExportJobOptions{ - Path: "grafana", - }, - }, - } + mockRepo := repository.NewMockRepository(t) + mockRepo.On("Config").Return(&v0alpha1.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Spec: v0alpha1.RepositorySpec{ + Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow}, + }, + }) - mockRepo := tt.createRepo(t) - tt.setupRepo(mockRepo.MockClonedRepository) + mockProgress := jobs.NewMockJobProgressRecorder(t) + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("clone failed")) - scheme := runtime.NewScheme() - require.NoError(t, metav1.AddMetaToScheme(scheme)) - listGVK := schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: "FolderList", - } - listGVKDashboard := schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: "DashboardList", - } - - scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(listGVKDashboard, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.FolderResource.Group, - Version: resources.FolderResource.Version, - Kind: resources.FolderResource.Resource, - }, &metav1.PartialObjectMetadata{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: resources.DashboardResource.Resource, - }, &metav1.PartialObjectMetadata{}) - - fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - resources.FolderResource: listGVK.Kind, - resources.DashboardResource: listGVKDashboard.Kind, - }) - fakeFolderClient := fakeDynamicClient.Resource(resources.FolderResource) - - mockProgress := jobs.NewMockJobProgressRecorder(t) - tt.setupProgress(mockProgress) - - mockRepoResources := resources.NewMockRepositoryResourcesFactory(t) - mockClientFactory := resources.NewMockClientFactory(t) - var repoResources *resources.MockRepositoryResources - - if tt.setupResources != nil { - resourceClients := resources.NewMockResourceClients(t) - resourceClients.On("Folder").Return(fakeFolderClient, nil) + r := NewExportWorker(nil, nil, nil, mockCloneFn.Execute) + err := r.Process(context.Background(), mockRepo, job, mockProgress) + require.EqualError(t, err, "clone failed") +} + +func TestExportWorker_ProcessBranchNotAllowedForClonableRepositories(t *testing.T) { + job := v0alpha1.Job{ + Spec: v0alpha1.JobSpec{ + Action: v0alpha1.JobActionPush, + Push: &v0alpha1.ExportJobOptions{ + Branch: "somebranch", + }, + }, + } - fakeDynamicClient.PrependReactor("list", "folders", tt.reactorFunc) - fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc) - mockClientFactory.On("Clients", mock.Anything, "test-namespace").Return(resourceClients, nil) - - repoResources = resources.NewMockRepositoryResources(t) - tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVKDashboard) - mockRepoResources.On("Client", mock.Anything, mock.MatchedBy(func(repo repository.ReaderWriter) bool { - // compare only pointers - return repo == mockRepo.MockClonedRepository - })).Return(repoResources, nil) - } - - r := NewExportWorker(mockClientFactory, mockRepoResources) - err := r.Process(context.Background(), mockRepo, job, mockProgress) - - if tt.expectedError != "" { - require.EqualError(t, err, tt.expectedError) - } else { - require.NoError(t, err) - } - - mockProgress.AssertExpectations(t) - mockRepoResources.AssertExpectations(t) - mockClientFactory.AssertExpectations(t) - - if repoResources != nil { - repoResources.AssertExpectations(t) - } - }) - } + mockRepo := repository.NewMockRepository(t) + mockRepo.On("Config").Return(&v0alpha1.Repository{ + Spec: v0alpha1.RepositorySpec{ + Type: v0alpha1.GitHubRepositoryType, + Workflows: []v0alpha1.Workflow{v0alpha1.BranchWorkflow}, + }, + }) + + mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("SetMessage", mock.Anything, "clone target").Return() + mockCloneFn := NewMockWrapWithCloneFn(t) + mockCloneFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, cloneOpts repository.CloneOptions, pushOpts repository.PushOptions, fn func(repository.Repository, bool) error) error { + if cloneOpts.BeforeFn != nil { + return cloneOpts.BeforeFn() + } + + return fn(repo, true) + }) + + r := NewExportWorker(nil, nil, nil, mockCloneFn.Execute) + err := r.Process(context.Background(), mockRepo, job, mockProgress) + require.EqualError(t, err, "branch is not supported for clonable repositories") } diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index f8a0b5617b7..8f737d37b15 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -547,7 +547,10 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH exportWorker := export.NewExportWorker( b.clients, b.repositoryResources, + export.ExportAll, + repository.WrapWithCloneAndPushIfPossible, ) + syncWorker := sync.NewSyncWorker( b.GetClient(), b.parsers,